From a18ad067088ab91c5ed4385807df45efb68b12d6 Mon Sep 17 00:00:00 2001 From: ArchieAlexArkhipov Date: Sat, 6 May 2023 20:00:49 +0000 Subject: [PATCH 01/25] dla --- mmdet/models/backbones/__init__.py | 28 +- mmdet/models/backbones/dla_custom.py | 634 ++++++++++++++++++++++++++ mmdet/models/backbones/dla_mmdet3d.py | 477 +++++++++++++++++++ mmdet/models/necks/dla_neck.py | 250 ++++++++++ 4 files changed, 1384 insertions(+), 5 deletions(-) create mode 100644 mmdet/models/backbones/dla_custom.py create mode 100644 mmdet/models/backbones/dla_mmdet3d.py create mode 100644 mmdet/models/necks/dla_neck.py diff --git a/mmdet/models/backbones/__init__.py b/mmdet/models/backbones/__init__.py index 91b50d25..09cd7722 100644 --- a/mmdet/models/backbones/__init__.py +++ b/mmdet/models/backbones/__init__.py @@ -3,6 +3,8 @@ from .darknet import Darknet from .detectors_resnet import DetectoRS_ResNet from .detectors_resnext import DetectoRS_ResNeXt +from .dla_custom import DLANetCustom +from .dla_mmdet3d import DLANetMMDet3D from .efficientnet import EfficientNet from .hourglass import HourglassNet from .hrnet import HRNet @@ -18,9 +20,25 @@ from .trident_resnet import TridentResNet __all__ = [ - 'RegNet', 'ResNet', 'ResNetV1d', 'ResNeXt', 'SSDVGG', 'HRNet', - 'MobileNetV2', 'Res2Net', 'HourglassNet', 'DetectoRS_ResNet', - 'DetectoRS_ResNeXt', 'Darknet', 'ResNeSt', 'TridentResNet', 'CSPDarknet', - 'SwinTransformer', 'PyramidVisionTransformer', - 'PyramidVisionTransformerV2', 'EfficientNet' + "RegNet", + "ResNet", + "ResNetV1d", + "ResNeXt", + "SSDVGG", + "HRNet", + "MobileNetV2", + "Res2Net", + "HourglassNet", + "DetectoRS_ResNet", + "DetectoRS_ResNeXt", + "Darknet", + "ResNeSt", + "TridentResNet", + "CSPDarknet", + "SwinTransformer", + "PyramidVisionTransformer", + "PyramidVisionTransformerV2", + "EfficientNet", + "DLANetCustom", + "DLANetMMDet3D", ] diff --git a/mmdet/models/backbones/dla_custom.py b/mmdet/models/backbones/dla_custom.py new file mode 100644 index 00000000..ebf8a12a --- /dev/null +++ b/mmdet/models/backbones/dla_custom.py @@ -0,0 +1,634 @@ +import torch +import torch.utils.checkpoint as cp +from mmcv.cnn import build_conv_layer, build_norm_layer, constant_init +from mmcv.runner import load_checkpoint +from mmdet.utils import get_root_logger +from torch import nn +from torch.nn.modules.batchnorm import _BatchNorm + +from ..builder import BACKBONES +from .resnet import BasicBlock as _BasicBlock +from .resnet import Bottleneck as _Bottleneck +from .resnet import ResNet + + +class BasicBlock(_BasicBlock): + def __init__( + self, + inplanes, + planes, + stride=1, + dilation=1, + downsample=None, + style="pytorch", + with_cp=False, + conv_cfg=None, + norm_cfg=dict(type="BN"), + dcn=None, + plugins=None, + ): + super(BasicBlock, self).__init__( + inplanes, + planes, + stride=stride, + dilation=dilation, + downsample=downsample, + style=style, + with_cp=with_cp, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + dcn=dcn, + plugins=plugins, + ) + + def forward(self, x, identity=None): + """Forward function.""" + + def _inner_forward(x, identity): + + if identity is None: + identity = x + + out = self.conv1(x) + out = self.norm1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.norm2(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + + return out + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(_inner_forward, x, identity) + else: + out = _inner_forward(x, identity) + + out = self.relu(out) + + return out + + +class Bottleneck(_Bottleneck): + expansion = 2 + + def __init__( + self, + inplanes, + planes, + stride=1, + dilation=1, + downsample=None, + style="pytorch", + with_cp=False, + conv_cfg=None, + norm_cfg=dict(type="BN"), + dcn=None, + plugins=None, + ): + super(Bottleneck, self).__init__( + inplanes, + planes, + stride=stride, + dilation=dilation, + downsample=downsample, + style=style, + with_cp=with_cp, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + dcn=dcn, + plugins=plugins, + ) + """Bottleneck block for DLA.""" + + expansion = self.expansion + bottle_planes = planes // expansion + + self.norm1_name, norm1 = build_norm_layer( + norm_cfg, bottle_planes, postfix=1 + ) + self.norm2_name, norm2 = build_norm_layer( + norm_cfg, bottle_planes, postfix=2 + ) + self.norm3_name, norm3 = build_norm_layer(norm_cfg, planes, postfix=3) + + self.conv1 = build_conv_layer( + conv_cfg, + inplanes, + bottle_planes, + kernel_size=1, + stride=self.conv1_stride, + bias=False, + ) + + self.add_module(self.norm1_name, norm1) + fallback_on_stride = False + if self.with_dcn: + fallback_on_stride = dcn.pop("fallback_on_stride", False) + if not self.with_dcn or fallback_on_stride: + self.conv2 = build_conv_layer( + conv_cfg, + bottle_planes, + bottle_planes, + kernel_size=3, + stride=self.conv2_stride, + padding=dilation, + dilation=dilation, + bias=False, + ) + else: + assert self.conv_cfg is None, "conv_cfg must be None for DCN" + self.conv2 = build_conv_layer( + dcn, + bottle_planes, + bottle_planes, + kernel_size=3, + stride=self.conv2_stride, + padding=dilation, + dilation=dilation, + bias=False, + ) + self.add_module(self.norm2_name, norm2) + self.conv3 = build_conv_layer( + conv_cfg, bottle_planes, planes, kernel_size=1, bias=False + ) + self.add_module(self.norm3_name, norm3) + + if self.with_plugins: + for name in self.after_conv1_plugin_names: + delattr(self, name) + for name in self.after_conv2_plugin_names: + delattr(self, name) + for name in self.after_conv3_plugin_names: + delattr(self, name) + self.after_conv1_plugin_names = self.make_block_plugins( + bottle_planes, self.after_conv1_plugins + ) + self.after_conv2_plugin_names = self.make_block_plugins( + bottle_planes, self.after_conv2_plugins + ) + self.after_conv3_plugin_names = self.make_block_plugins( + planes, self.after_conv3_plugins + ) + + def forward(self, x, identity=None): + """Forward function.""" + + def _inner_forward(x, identity): + if identity is None: + identity = x + + out = self.conv1(x) + out = self.norm1(out) + out = self.relu(out) + + if self.with_plugins: + out = self.forward_plugin(out, self.after_conv1_plugin_names) + + out = self.conv2(out) + out = self.norm2(out) + out = self.relu(out) + + if self.with_plugins: + out = self.forward_plugin(out, self.after_conv2_plugin_names) + + out = self.conv3(out) + out = self.norm3(out) + + if self.with_plugins: + out = self.forward_plugin(out, self.after_conv3_plugin_names) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + + return out + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(_inner_forward, x, identity) + else: + out = _inner_forward(x, identity) + + out = self.relu(out) + + return out + + +class Root(nn.Module): + def __init__( + self, + in_channels, + out_channels, + kernel_size, + residual, + conv_cfg=None, + norm_cfg=dict(type="BN"), + ): + super(Root, self).__init__() + self.conv = build_conv_layer( + conv_cfg, + in_channels, + out_channels, + kernel_size, + stride=1, + bias=False, + padding=(kernel_size - 1) // 2, + ) + self.norm_name, norm = build_norm_layer(norm_cfg, out_channels) + self.add_module(self.norm_name, norm) + self.relu = nn.ReLU(inplace=True) + self.residual = residual + + @property + def norm(self): + """nn.Module: the normalization layer named "norm" """ + return getattr(self, self.norm_name) + + def forward(self, *x): + children = x + x = self.conv(torch.cat(x, 1)) + x = self.norm(x) + if self.residual: + x += children[0] + x = self.relu(x) + + return x + + +class Tree(nn.Module): + def __init__( + self, + levels, + block, + in_channels, + out_channels, + stride=1, + level_root=False, + root_dim=0, + root_kernel_size=1, + dilation=1, + root_residual=False, + conv_cfg=None, + norm_cfg=dict(type="BN"), + with_cp=False, + dcn=None, + plugins=None, + style="pytorch", + ): + super(Tree, self).__init__() + if root_dim == 0: + root_dim = 2 * out_channels + if level_root: + root_dim += in_channels + if levels == 1: + self.tree1 = block( + in_channels, + out_channels, + stride, + dilation=dilation, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + with_cp=with_cp, + dcn=dcn, + plugins=plugins, + style=style, + ) + self.tree2 = block( + out_channels, + out_channels, + 1, + dilation=dilation, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + with_cp=with_cp, + dcn=dcn, + plugins=plugins, + style=style, + ) + self.root = Root( + root_dim, + out_channels, + root_kernel_size, + root_residual, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + ) + else: + self.tree1 = Tree( + levels - 1, + block, + in_channels, + out_channels, + stride, + root_dim=0, + root_kernel_size=root_kernel_size, + dilation=dilation, + root_residual=root_residual, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + with_cp=with_cp, + dcn=dcn, + plugins=plugins, + style=style, + ) + self.tree2 = Tree( + levels - 1, + block, + out_channels, + out_channels, + root_dim=root_dim + out_channels, + root_kernel_size=root_kernel_size, + dilation=dilation, + root_residual=root_residual, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + with_cp=with_cp, + dcn=dcn, + plugins=plugins, + style=style, + ) + + self.level_root = level_root + self.downsample = None + self.project = None + self.levels = levels + if stride > 1: + self.downsample = nn.MaxPool2d(stride, stride=stride) + if in_channels != out_channels and self.levels == 1: + self.project = nn.Sequential( + build_conv_layer( + conv_cfg, + in_channels, + out_channels, + kernel_size=1, + stride=1, + bias=False, + ), + build_norm_layer(norm_cfg, out_channels)[1], + ) + + def forward(self, x, residual=None, children=None): + children = [] if children is None else children + bottom = self.downsample(x) if self.downsample else x + residual = self.project(bottom) if self.project else bottom + if self.level_root: + children.append(bottom) + x1 = self.tree1(x, residual) + if self.levels == 1: + x2 = self.tree2(x1) + x = self.root(x2, x1, *children) + else: + children.append(x1) + x = self.tree2(x1, children=children) + return x + + +@BACKBONES.register_module() +class DLANetCustom(nn.Module): + """DLA backbone. + + Args: + depth (int): Depth of dla, from {34, 46, 60, 102, 169}. + in_channels (int): Number of input image channels. Default: 3. + num_stages (int): dla stages. Default: 4. + out_indices (Sequence[int]): Output from which stages. + strides (Sequence[int]): Strides of the first block of each stage. + style (str): `pytorch` or `caffe`. If set to "pytorch", the stride-two + layer is the 3x3 conv layer, otherwise the stride-two layer is + the first 1x1 conv layer. + frozen_stages (int): Stages to be frozen (stop grad and set eval mode). + -1 means not freezing any parameters. + conv_cfg (dict): Dictionary to construct and config convolution layer. + norm_cfg (dict): Dictionary to construct and config norm layer. + norm_eval (bool): Whether to set norm layers to eval mode, namely, + freeze running stats (mean and var). Note: Effect on Batch Norm + and its variants only. + dcn (dict): Dictionary to construct and config deformable convolution + layer + stage_with_dcn (tuple[bool]): Stages to apply dcn, length + should be same as 'num_stages'. + plugins (list[dict]): List of plugins for stages, each dict contains: + + - cfg (dict, required): Cfg dict to build plugin. + - position (str, required): Position inside block to insert + plugin, options are 'after_conv1', 'after_conv2', 'after_conv3'. + - stages (tuple[bool], optional): Stages to apply plugin, length + should be same as 'num_stages'. + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. + zero_init_residual (bool): Whether to use zero init for last norm layer + in resblocks to let them behave as identity. + stage_with_level_root (tuple[bool]): Stages to apply level_root, length + should be same as 'num_stages'. + residual_root (bool): whether to use residual in root layer + + Example: + >>> from mmdet.models import DLANet + >>> import torch + >>> self = DLANet(depth=60) + >>> self.eval() + >>> inputs = torch.rand(1, 3, 32, 32) + >>> level_outputs = self.forward(inputs) + >>> for level_out in level_outputs: + ... print(tuple(level_out.shape)) + (1, 128, 8, 8) + (1, 256, 4, 4) + (1, 512, 2, 2) + (1, 1024, 1, 1) + """ + + arch_settings = { + 34: (BasicBlock, [1, 1, 1, 2, 2, 1], [16, 32, 64, 128, 256, 512]), + 46: (Bottleneck, [1, 1, 1, 2, 2, 1], [16, 32, 64, 64, 128, 256]), + 60: (Bottleneck, [1, 1, 1, 2, 3, 1], [16, 32, 128, 256, 512, 1024]), + 102: (Bottleneck, [1, 1, 1, 3, 4, 1], [16, 32, 128, 256, 512, 1024]), + 169: (Bottleneck, [1, 1, 2, 3, 5, 1], [16, 32, 128, 256, 512, 1024]), + } + + make_stage_plugins = ResNet.make_stage_plugins + + def __init__( + self, + depth, + in_channels=3, + num_stages=4, + out_indices=(0, 1, 2, 3), + strides=(2, 2, 2, 2), + style="pytorch", + frozen_stages=-1, + conv_cfg=None, + norm_cfg=dict(type="BN", requires_grad=True), + norm_eval=True, + dcn=None, + stage_with_dcn=(False, False, False, False), + plugins=None, + with_cp=False, + zero_init_residual=True, + stage_with_level_root=(False, True, True, True), + residual_root=False, + ): + super(DLANetCustom, self).__init__() + if depth not in self.arch_settings: + raise KeyError(f"invalid depth {depth} for DLA") + block, levels, channels = self.arch_settings[depth] + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.zero_init_residual = zero_init_residual + self.frozen_stages = frozen_stages + self.num_stages = num_stages + assert num_stages >= 1 and num_stages <= 4 + self.out_indices = out_indices + assert max(out_indices) < num_stages + self.style = style + self.with_cp = with_cp + self.norm_eval = norm_eval + self.dcn = dcn + self.stage_with_dcn = stage_with_dcn + self.base_layer = nn.Sequential( + build_conv_layer( + self.conv_cfg, + in_channels, + channels[0], + kernel_size=7, + stride=1, + padding=3, + bias=False, + ), + build_norm_layer(self.norm_cfg, channels[0])[1], + nn.ReLU(inplace=True), + ) + + for i in range(2): + level_layer = self._make_conv_level( + channels[0], channels[i], levels[i], stride=i + 1 + ) + layer_name = f"level{i}" + self.add_module(layer_name, level_layer) + + for i in range(self.num_stages): + dcn = self.dcn if self.stage_with_dcn[i] else None + if plugins is not None: + stage_plugins = self.make_stage_plugins(plugins, i) + else: + stage_plugins = None + dla_layer = Tree( + levels[i + 2], + block, + channels[i + 1], + channels[i + 2], + strides[i], + level_root=stage_with_level_root[i], + root_residual=residual_root, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + dcn=dcn, + plugins=stage_plugins, + style=self.style, + ) + layer_name = f"layer{i + 1}" + self.add_module(layer_name, dla_layer) + + self._freeze_stages() + + def _make_conv_level(self, inplanes, planes, convs, stride=1, dilation=1): + modules = [] + for i in range(convs): + modules.extend( + [ + build_conv_layer( + self.conv_cfg, + inplanes, + planes, + kernel_size=3, + stride=stride if i == 0 else 1, + padding=dilation, + bias=False, + dilation=dilation, + ), + build_norm_layer(self.norm_cfg, planes)[1], + nn.ReLU(inplace=True), + ] + ) + inplanes = planes + return nn.Sequential(*modules) + + def _freeze_stages(self): + if self.frozen_stages >= 0: + self.base_layer.eval() + for param in self.base_layer.parameters(): + param.requires_grad = False + + for i in range(2): + m = getattr(self, f"level{i}") + m.eval() + for param in m.parameters(): + param.requires_grad = False + + for i in range(1, self.frozen_stages + 1): + m = getattr(self, f"layer{i}") + m.eval() + for param in m.parameters(): + param.requires_grad = False + + def init_weights(self, pretrained=None): + """Initialize the weights in backbone. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + if isinstance(pretrained, str): + logger = get_root_logger() + load_checkpoint(self, pretrained, strict=False, logger=logger) + elif pretrained is None: + for m in self.modules(): + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(0, torch.tensor(2.0 / n).sqrt()) + elif isinstance(m, (_BatchNorm, nn.GroupNorm)): + m.weight.data.fill_(1) + m.bias.data.zero_() + + if self.dcn is not None: + for m in self.modules(): + if isinstance(m, Bottleneck) and hasattr( + m.conv2, "conv_offset" + ): + constant_init(m.conv2.conv_offset, 0) + + if self.zero_init_residual: + for m in self.modules(): + if isinstance(m, Bottleneck): + constant_init(m.norm3, 0) + elif isinstance(m, BasicBlock): + constant_init(m.norm2, 0) + else: + raise TypeError("pretrained must be a str or None") + + def forward(self, x): + """Forward function.""" + x = self.base_layer(x) + for i in range(2): + x = getattr(self, "level{}".format(i))(x) + outs = [] + for i, index in enumerate(range(1, self.num_stages + 1)): + x = getattr(self, "layer{}".format(index))(x) + if i in self.out_indices: + outs.append(x) + return tuple(outs) + + def train(self, mode=True): + """Convert the model into training mode while keep normalization layer + freezed.""" + super(DLANetCustom, self).train(mode) + self._freeze_stages() + if mode and self.norm_eval: + for m in self.modules(): + # trick: eval have effect on BatchNorm only + if isinstance(m, _BatchNorm): + m.eval() diff --git a/mmdet/models/backbones/dla_mmdet3d.py b/mmdet/models/backbones/dla_mmdet3d.py new file mode 100644 index 00000000..2f0b0965 --- /dev/null +++ b/mmdet/models/backbones/dla_mmdet3d.py @@ -0,0 +1,477 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import warnings + +import torch +from mmcv.cnn import build_conv_layer, build_norm_layer +from mmcv.runner import BaseModule +from torch import nn + +from ..builder import BACKBONES + + +def dla_build_norm_layer(cfg, num_features): + """Build normalization layer specially designed for DLANet. + + Args: + cfg (dict): The norm layer config, which should contain: + + - type (str): Layer type. + - layer args: Args needed to instantiate a norm layer. + - requires_grad (bool, optional): Whether stop gradient updates. + num_features (int): Number of input channels. + + + Returns: + Function: Build normalization layer in mmcv. + """ + cfg_ = cfg.copy() + if cfg_["type"] == "GN": + if num_features % 32 == 0: + return build_norm_layer(cfg_, num_features) + else: + assert "num_groups" in cfg_ + cfg_["num_groups"] = cfg_["num_groups"] // 2 + return build_norm_layer(cfg_, num_features) + else: + return build_norm_layer(cfg_, num_features) + + +class BasicBlock(BaseModule): + """BasicBlock in DLANet. + + Args: + in_channels (int): Input feature channel. + out_channels (int): Output feature channel. + norm_cfg (dict): Dictionary to construct and config + norm layer. + conv_cfg (dict): Dictionary to construct and config + conv layer. + stride (int, optional): Conv stride. Default: 1. + dilation (int, optional): Conv dilation. Default: 1. + init_cfg (dict, optional): Initialization config. + Default: None. + """ + + def __init__( + self, + in_channels, + out_channels, + norm_cfg, + conv_cfg, + stride=1, + dilation=1, + init_cfg=None, + ): + super(BasicBlock, self).__init__(init_cfg) + self.conv1 = build_conv_layer( + conv_cfg, + in_channels, + out_channels, + 3, + stride=stride, + padding=dilation, + dilation=dilation, + bias=False, + ) + self.norm1 = dla_build_norm_layer(norm_cfg, out_channels)[1] + self.relu = nn.ReLU(inplace=True) + self.conv2 = build_conv_layer( + conv_cfg, + out_channels, + out_channels, + 3, + stride=1, + padding=dilation, + dilation=dilation, + bias=False, + ) + self.norm2 = dla_build_norm_layer(norm_cfg, out_channels)[1] + self.stride = stride + + def forward(self, x, identity=None): + """Forward function.""" + + if identity is None: + identity = x + out = self.conv1(x) + out = self.norm1(out) + out = self.relu(out) + out = self.conv2(out) + out = self.norm2(out) + out += identity + out = self.relu(out) + + return out + + +class Root(BaseModule): + """Root in DLANet. + + Args: + in_channels (int): Input feature channel. + out_channels (int): Output feature channel. + norm_cfg (dict): Dictionary to construct and config + norm layer. + conv_cfg (dict): Dictionary to construct and config + conv layer. + kernel_size (int): Size of convolution kernel. + add_identity (bool): Whether to add identity in root. + init_cfg (dict, optional): Initialization config. + Default: None. + """ + + def __init__( + self, + in_channels, + out_channels, + norm_cfg, + conv_cfg, + kernel_size, + add_identity, + init_cfg=None, + ): + super(Root, self).__init__(init_cfg) + self.conv = build_conv_layer( + conv_cfg, + in_channels, + out_channels, + 1, + stride=1, + padding=(kernel_size - 1) // 2, + bias=False, + ) + self.norm = dla_build_norm_layer(norm_cfg, out_channels)[1] + self.relu = nn.ReLU(inplace=True) + self.add_identity = add_identity + + def forward(self, feat_list): + """Forward function. + + Args: + feat_list (list[torch.Tensor]): Output features from + multiple layers. + """ + children = feat_list + x = self.conv(torch.cat(feat_list, 1)) + x = self.norm(x) + if self.add_identity: + x += children[0] + x = self.relu(x) + + return x + + +class Tree(BaseModule): + """Tree in DLANet. + + Args: + levels (int): The level of the tree. + block (nn.Module): The block module in tree. + in_channels: Input feature channel. + out_channels: Output feature channel. + norm_cfg (dict): Dictionary to construct and config + norm layer. + conv_cfg (dict): Dictionary to construct and config + conv layer. + stride (int, optional): Convolution stride. + Default: 1. + level_root (bool, optional): whether belongs to the + root layer. + root_dim (int, optional): Root input feature channel. + root_kernel_size (int, optional): Size of root + convolution kernel. Default: 1. + dilation (int, optional): Conv dilation. Default: 1. + add_identity (bool, optional): Whether to add + identity in root. Default: False. + init_cfg (dict, optional): Initialization config. + Default: None. + """ + + def __init__( + self, + levels, + block, + in_channels, + out_channels, + norm_cfg, + conv_cfg, + stride=1, + level_root=False, + root_dim=None, + root_kernel_size=1, + dilation=1, + add_identity=False, + init_cfg=None, + ): + super(Tree, self).__init__(init_cfg) + if root_dim is None: + root_dim = 2 * out_channels + if level_root: + root_dim += in_channels + if levels == 1: + self.root = Root( + root_dim, + out_channels, + norm_cfg, + conv_cfg, + root_kernel_size, + add_identity, + ) + self.tree1 = block( + in_channels, + out_channels, + norm_cfg, + conv_cfg, + stride, + dilation=dilation, + ) + self.tree2 = block( + out_channels, + out_channels, + norm_cfg, + conv_cfg, + 1, + dilation=dilation, + ) + else: + self.tree1 = Tree( + levels - 1, + block, + in_channels, + out_channels, + norm_cfg, + conv_cfg, + stride, + root_dim=None, + root_kernel_size=root_kernel_size, + dilation=dilation, + add_identity=add_identity, + ) + self.tree2 = Tree( + levels - 1, + block, + out_channels, + out_channels, + norm_cfg, + conv_cfg, + root_dim=root_dim + out_channels, + root_kernel_size=root_kernel_size, + dilation=dilation, + add_identity=add_identity, + ) + self.level_root = level_root + self.root_dim = root_dim + self.downsample = None + self.project = None + self.levels = levels + if stride > 1: + self.downsample = nn.MaxPool2d(stride, stride=stride) + if in_channels != out_channels: + self.project = nn.Sequential( + build_conv_layer( + conv_cfg, in_channels, out_channels, 1, stride=1, bias=False + ), + dla_build_norm_layer(norm_cfg, out_channels)[1], + ) + + def forward(self, x, identity=None, children=None): + children = [] if children is None else children + bottom = self.downsample(x) if self.downsample else x + identity = self.project(bottom) if self.project else bottom + if self.level_root: + children.append(bottom) + x1 = self.tree1(x, identity) + if self.levels == 1: + x2 = self.tree2(x1) + feat_list = [x2, x1] + children + x = self.root(feat_list) + else: + children.append(x1) + x = self.tree2(x1, children=children) + return x + + +@BACKBONES.register_module() +class DLANetMMDet3D(BaseModule): + r"""`DLA backbone `_. + + Args: + depth (int): Depth of DLA. Default: 34. + in_channels (int, optional): Number of input image channels. + Default: 3. + norm_cfg (dict, optional): Dictionary to construct and config + norm layer. Default: None. + conv_cfg (dict, optional): Dictionary to construct and config + conv layer. Default: None. + layer_with_level_root (list[bool], optional): Whether to apply + level_root in each DLA layer, this is only used for + tree levels. Default: (False, True, True, True). + with_identity_root (bool, optional): Whether to add identity + in root layer. Default: False. + pretrained (str, optional): model pretrained path. + Default: None. + init_cfg (dict or list[dict], optional): Initialization + config dict. Default: None + """ + arch_settings = { + 34: (BasicBlock, (1, 1, 1, 2, 2, 1), (16, 32, 64, 128, 256, 512)), + } + + def __init__( + self, + depth, + in_channels=3, + out_indices=(0, 1, 2, 3, 4, 5), + frozen_stages=-1, + norm_cfg=None, + conv_cfg=None, + layer_with_level_root=(False, True, True, True), + with_identity_root=False, + pretrained=None, + init_cfg=None, + ): + super(DLANetMMDet3D, self).__init__(init_cfg) + if depth not in self.arch_settings: + raise KeyError(f"invalida depth {depth} for DLA") + + assert not ( + init_cfg and pretrained + ), "init_cfg and pretrained cannot be setting at the same time" + if isinstance(pretrained, str): + warnings.warn( + "DeprecationWarning: pretrained is a deprecated, " + 'please use "init_cfg" instead' + ) + self.init_cfg = dict(type="Pretrained", checkpoint=pretrained) + elif pretrained is None: + if init_cfg is None: + self.init_cfg = [ + dict(type="Kaiming", layer="Conv2d"), + dict( + type="Constant", + val=1, + layer=["_BatchNorm", "GroupNorm"], + ), + ] + + block, levels, channels = self.arch_settings[depth] + self.channels = channels + self.num_levels = len(levels) + self.frozen_stages = frozen_stages + self.out_indices = out_indices + assert max(out_indices) < self.num_levels + self.base_layer = nn.Sequential( + build_conv_layer( + conv_cfg, + in_channels, + channels[0], + 7, + stride=1, + padding=3, + bias=False, + ), + dla_build_norm_layer(norm_cfg, channels[0])[1], + nn.ReLU(inplace=True), + ) + + # DLANet first uses two conv layers then uses several + # Tree layers + for i in range(2): + level_layer = self._make_conv_level( + channels[0], + channels[i], + levels[i], + norm_cfg, + conv_cfg, + stride=i + 1, + ) + layer_name = f"level{i}" + self.add_module(layer_name, level_layer) + + for i in range(2, self.num_levels): + dla_layer = Tree( + levels[i], + block, + channels[i - 1], + channels[i], + norm_cfg, + conv_cfg, + 2, + level_root=layer_with_level_root[i - 2], + add_identity=with_identity_root, + ) + layer_name = f"level{i}" + self.add_module(layer_name, dla_layer) + + self._freeze_stages() + + def _make_conv_level( + self, + in_channels, + out_channels, + num_convs, + norm_cfg, + conv_cfg, + stride=1, + dilation=1, + ): + """Conv modules. + + Args: + in_channels (int): Input feature channel. + out_channels (int): Output feature channel. + num_convs (int): Number of Conv module. + norm_cfg (dict): Dictionary to construct and config + norm layer. + conv_cfg (dict): Dictionary to construct and config + conv layer. + stride (int, optional): Conv stride. Default: 1. + dilation (int, optional): Conv dilation. Default: 1. + """ + modules = [] + for i in range(num_convs): + modules.extend( + [ + build_conv_layer( + conv_cfg, + in_channels, + out_channels, + 3, + stride=stride if i == 0 else 1, + padding=dilation, + bias=False, + dilation=dilation, + ), + dla_build_norm_layer(norm_cfg, out_channels)[1], + nn.ReLU(inplace=True), + ] + ) + in_channels = out_channels + return nn.Sequential(*modules) + + def _freeze_stages(self): + if self.frozen_stages >= 0: + self.base_layer.eval() + for param in self.base_layer.parameters(): + param.requires_grad = False + + for i in range(2): + m = getattr(self, f"level{i}") + m.eval() + for param in m.parameters(): + param.requires_grad = False + + for i in range(1, self.frozen_stages + 1): + m = getattr(self, f"level{i+1}") + m.eval() + for param in m.parameters(): + param.requires_grad = False + + def forward(self, x): + outs = [] + x = self.base_layer(x) + for i in range(self.num_levels): + x = getattr(self, "level{}".format(i))(x) + if i in self.out_indices: + outs.append(x) + return tuple(outs) diff --git a/mmdet/models/necks/dla_neck.py b/mmdet/models/necks/dla_neck.py new file mode 100644 index 00000000..ad520033 --- /dev/null +++ b/mmdet/models/necks/dla_neck.py @@ -0,0 +1,250 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import math + +import numpy as np +from mmcv.cnn import ConvModule, build_conv_layer +from mmcv.runner import BaseModule +from torch import nn as nn + +from ..builder import NECKS + + +def fill_up_weights(up): + """Simulated bilinear upsampling kernel. + + Args: + up (nn.Module): ConvTranspose2d module. + """ + w = up.weight.data + f = math.ceil(w.size(2) / 2) + c = (2 * f - 1 - f % 2) / (2.0 * f) + for i in range(w.size(2)): + for j in range(w.size(3)): + w[0, 0, i, j] = (1 - math.fabs(i / f - c)) * ( + 1 - math.fabs(j / f - c) + ) + for c in range(1, w.size(0)): + w[c, 0, :, :] = w[0, 0, :, :] + + +class IDAUpsample(BaseModule): + """Iterative Deep Aggregation (IDA) Upsampling module to upsample features + of different scales to a similar scale. + + Args: + out_channels (int): Number of output channels for DeformConv. + in_channels (List[int]): List of input channels of multi-scale + feature maps. + kernel_sizes (List[int]): List of size of the convolving + kernel of different scales. + norm_cfg (dict, optional): Config dict for normalization layer. + Default: None. + use_dcn (bool, optional): If True, use DCNv2. Default: True. + """ + + def __init__( + self, + out_channels, + in_channels, + kernel_sizes, + norm_cfg=None, + use_dcn=True, + init_cfg=None, + ): + super(IDAUpsample, self).__init__(init_cfg) + self.use_dcn = use_dcn + self.projs = nn.ModuleList() + self.ups = nn.ModuleList() + self.nodes = nn.ModuleList() + + for i in range(1, len(in_channels)): + in_channel = in_channels[i] + up_kernel_size = int(kernel_sizes[i]) + proj = ConvModule( + in_channel, + out_channels, + 3, + padding=1, + bias=True, + conv_cfg=dict(type="DCNv2") if self.use_dcn else None, + norm_cfg=norm_cfg, + ) + node = ConvModule( + out_channels, + out_channels, + 3, + padding=1, + bias=True, + conv_cfg=dict(type="DCNv2") if self.use_dcn else None, + norm_cfg=norm_cfg, + ) + up = build_conv_layer( + dict(type="deconv"), + out_channels, + out_channels, + up_kernel_size * 2, + stride=up_kernel_size, + padding=up_kernel_size // 2, + output_padding=0, + groups=out_channels, + bias=False, + ) + + self.projs.append(proj) + self.ups.append(up) + self.nodes.append(node) + + def forward(self, mlvl_features, start_level, end_level): + """Forward function. + + Args: + mlvl_features (list[torch.Tensor]): Features from multiple layers. + start_level (int): Start layer for feature upsampling. + end_level (int): End layer for feature upsampling. + """ + for i in range(start_level, end_level - 1): + upsample = self.ups[i - start_level] + project = self.projs[i - start_level] + mlvl_features[i + 1] = upsample(project(mlvl_features[i + 1])) + node = self.nodes[i - start_level] + mlvl_features[i + 1] = node(mlvl_features[i + 1] + mlvl_features[i]) + + +class DLAUpsample(BaseModule): + """Deep Layer Aggregation (DLA) Upsampling module for different scales + feature extraction, upsampling and fusion, It consists of groups of + IDAupsample modules. + + Args: + start_level (int): The start layer. + channels (List[int]): List of input channels of multi-scale + feature maps. + scales(List[int]): List of scale of different layers' feature. + in_channels (NoneType, optional): List of input channels of + different scales. Default: None. + norm_cfg (dict, optional): Config dict for normalization layer. + Default: None. + use_dcn (bool, optional): Whether to use dcn in IDAup module. + Default: True. + """ + + def __init__( + self, + start_level, + channels, + scales, + in_channels=None, + norm_cfg=None, + use_dcn=True, + init_cfg=None, + ): + super(DLAUpsample, self).__init__(init_cfg) + self.start_level = start_level + if in_channels is None: + in_channels = channels + self.channels = channels + channels = list(channels) + scales = np.array(scales, dtype=int) + for i in range(len(channels) - 1): + j = -i - 2 + setattr( + self, + "ida_{}".format(i), + IDAUpsample( + channels[j], + in_channels[j:], + scales[j:] // scales[j], + norm_cfg, + use_dcn, + ), + ) + scales[j + 1 :] = scales[j] + in_channels[j + 1 :] = [channels[j] for _ in channels[j + 1 :]] + + def forward(self, mlvl_features): + """Forward function. + + Args: + mlvl_features(list[torch.Tensor]): Features from multi-scale + layers. + + Returns: + tuple[torch.Tensor]: Up-sampled features of different layers. + """ + outs = [mlvl_features[-1]] + for i in range(len(mlvl_features) - self.start_level - 1): + ida = getattr(self, "ida_{}".format(i)) + ida(mlvl_features, len(mlvl_features) - i - 2, len(mlvl_features)) + outs.insert(0, mlvl_features[-1]) + return outs + + +@NECKS.register_module() +class DLANeck(BaseModule): + """DLA Neck. + + Args: + in_channels (list[int], optional): List of input channels + of multi-scale feature map. + start_level (int, optional): The scale level where upsampling + starts. Default: 2. + end_level (int, optional): The scale level where upsampling + ends. Default: 5. + norm_cfg (dict, optional): Config dict for normalization + layer. Default: None. + use_dcn (bool, optional): Whether to use dcn in IDAup module. + Default: True. + """ + + def __init__( + self, + in_channels=[16, 32, 64, 128, 256, 512], + start_level=2, + end_level=5, + norm_cfg=None, + use_dcn=True, + init_cfg=None, + ): + super(DLANeck, self).__init__(init_cfg) + self.start_level = start_level + self.end_level = end_level + scales = [2**i for i in range(len(in_channels[self.start_level :]))] + self.dla_up = DLAUpsample( + start_level=self.start_level, + channels=in_channels[self.start_level :], + scales=scales, + norm_cfg=norm_cfg, + use_dcn=use_dcn, + ) + self.ida_up = IDAUpsample( + in_channels[self.start_level], + in_channels[self.start_level : self.end_level], + [2**i for i in range(self.end_level - self.start_level)], + norm_cfg, + use_dcn, + ) + + def forward(self, x): + mlvl_features = [x[i] for i in range(len(x))] + mlvl_features = self.dla_up(mlvl_features) + outs = [] + for i in range(self.end_level - self.start_level): + outs.append(mlvl_features[i].clone()) + self.ida_up(outs, 0, len(outs)) + return [outs[-1]] + + def init_weights(self): + for m in self.modules(): + if isinstance(m, nn.ConvTranspose2d): + # In order to be consistent with the source code, + # reset the ConvTranspose2d initialization parameters + m.reset_parameters() + # Simulated bilinear upsampling kernel + fill_up_weights(m) + elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.Conv2d): + # In order to be consistent with the source code, + # reset the Conv2d initialization parameters + m.reset_parameters() From 9fc43a8cfb77719e671ebfe0469b95124253fb68 Mon Sep 17 00:00:00 2001 From: ArchieAlexArkhipov Date: Sat, 6 May 2023 20:01:45 +0000 Subject: [PATCH 02/25] analyze result --- .../centernet_resnet18_dcnv2_140e_coco.py | 127 ++++++++----- tools/analysis_tools/analyze_results.py | 177 ++++++++++-------- 2 files changed, 169 insertions(+), 135 deletions(-) diff --git a/configs/centernet/centernet_resnet18_dcnv2_140e_coco.py b/configs/centernet/centernet_resnet18_dcnv2_140e_coco.py index b8a0bb10..67ae753c 100644 --- a/configs/centernet/centernet_resnet18_dcnv2_140e_coco.py +++ b/configs/centernet/centernet_resnet18_dcnv2_140e_coco.py @@ -1,92 +1,112 @@ _base_ = [ - '../_base_/datasets/coco_detection.py', - '../_base_/schedules/schedule_1x.py', '../_base_/default_runtime.py' + "../_base_/datasets/coco_detection.py", + "../_base_/schedules/schedule_1x.py", + "../_base_/default_runtime.py", ] model = dict( - type='CenterNet', + type="CenterNet", backbone=dict( - type='ResNet', + type="ResNet", depth=18, norm_eval=False, - norm_cfg=dict(type='BN'), - init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet18')), + norm_cfg=dict(type="BN"), + init_cfg=dict(type="Pretrained", checkpoint="torchvision://resnet18"), + ), neck=dict( - type='CTResNetNeck', + type="CTResNetNeck", in_channel=512, num_deconv_filters=(256, 128, 64), num_deconv_kernels=(4, 4, 4), - use_dcn=True), + use_dcn=True, + ), bbox_head=dict( - type='CenterNetHead', + type="CenterNetHead", num_classes=80, in_channel=64, feat_channel=64, - loss_center_heatmap=dict(type='GaussianFocalLoss', loss_weight=1.0), - loss_wh=dict(type='L1Loss', loss_weight=0.1), - loss_offset=dict(type='L1Loss', loss_weight=1.0)), + loss_center_heatmap=dict(type="GaussianFocalLoss", loss_weight=1.0), + loss_wh=dict(type="L1Loss", loss_weight=0.1), + loss_offset=dict(type="L1Loss", loss_weight=1.0), + ), train_cfg=None, - test_cfg=dict(topk=100, local_maximum_kernel=3, max_per_img=100)) + test_cfg=dict(topk=100, local_maximum_kernel=3, max_per_img=100), +) # We fixed the incorrect img_norm_cfg problem in the source code. img_norm_cfg = dict( - mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True +) train_pipeline = [ - dict(type='LoadImageFromFile', to_float32=True, color_type='color'), - dict(type='LoadAnnotations', with_bbox=True), + dict(type="LoadImageFromFile", to_float32=True, color_type="color"), + dict(type="LoadAnnotations", with_bbox=True), dict( - type='PhotoMetricDistortion', + type="PhotoMetricDistortion", brightness_delta=32, contrast_range=(0.5, 1.5), saturation_range=(0.5, 1.5), - hue_delta=18), + hue_delta=18, + ), dict( - type='RandomCenterCropPad', + type="RandomCenterCropPad", crop_size=(512, 512), ratios=(0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3), mean=[0, 0, 0], std=[1, 1, 1], to_rgb=True, - test_pad_mode=None), - dict(type='Resize', img_scale=(512, 512), keep_ratio=True), - dict(type='RandomFlip', flip_ratio=0.5), - dict(type='Normalize', **img_norm_cfg), - dict(type='DefaultFormatBundle'), - dict(type='Collect', keys=['img', 'gt_bboxes', 'gt_labels']) + test_pad_mode=None, + ), + dict(type="Resize", img_scale=(512, 512), keep_ratio=True), + dict(type="RandomFlip", flip_ratio=0.5), + dict(type="Normalize", **img_norm_cfg), + dict(type="DefaultFormatBundle"), + dict(type="Collect", keys=["img", "gt_bboxes", "gt_labels"]), ] test_pipeline = [ - dict(type='LoadImageFromFile', to_float32=True), + dict(type="LoadImageFromFile", to_float32=True), dict( - type='MultiScaleFlipAug', + type="MultiScaleFlipAug", scale_factor=1.0, flip=False, transforms=[ - dict(type='Resize', keep_ratio=True), + dict(type="Resize", keep_ratio=True), dict( - type='RandomCenterCropPad', + type="RandomCenterCropPad", ratios=None, border=None, mean=[0, 0, 0], std=[1, 1, 1], to_rgb=True, test_mode=True, - test_pad_mode=['logical_or', 31], - test_pad_add_pix=1), - dict(type='RandomFlip'), - dict(type='Normalize', **img_norm_cfg), - dict(type='DefaultFormatBundle'), + test_pad_mode=["logical_or", 31], + test_pad_add_pix=1, + ), + dict(type="RandomFlip"), + dict(type="Normalize", **img_norm_cfg), + dict(type="DefaultFormatBundle"), dict( - type='Collect', - meta_keys=('filename', 'ori_filename', 'ori_shape', - 'img_shape', 'pad_shape', 'scale_factor', 'flip', - 'flip_direction', 'img_norm_cfg', 'border'), - keys=['img']) - ]) + type="Collect", + meta_keys=( + "filename", + "ori_filename", + "ori_shape", + "img_shape", + "pad_shape", + "scale_factor", + "flip", + "flip_direction", + "img_norm_cfg", + "border", + ), + keys=["img"], + ), + ], + ), ] -dataset_type = 'CocoDataset' -data_root = 'data/coco/' +dataset_type = "CocoDataset" +data_root = "data/coco/" # Use RepeatDataset to speed up training data = dict( @@ -94,31 +114,34 @@ workers_per_gpu=4, train=dict( _delete_=True, - type='RepeatDataset', + type="RepeatDataset", times=5, dataset=dict( type=dataset_type, - ann_file=data_root + 'annotations/instances_train2017.json', - img_prefix=data_root + 'train2017/', - pipeline=train_pipeline)), + ann_file=data_root + "annotations/instances_train2017.json", + img_prefix=data_root + "train2017/", + pipeline=train_pipeline, + ), + ), val=dict(pipeline=test_pipeline), - test=dict(pipeline=test_pipeline)) + test=dict(pipeline=test_pipeline), +) # optimizer # Based on the default settings of modern detectors, the SGD effect is better # than the Adam in the source code, so we use SGD default settings and # if you use adam+lr5e-4, the map is 29.1. -optimizer_config = dict( - _delete_=True, grad_clip=dict(max_norm=35, norm_type=2)) +optimizer_config = dict(_delete_=True, grad_clip=dict(max_norm=35, norm_type=2)) # learning policy # Based on the default settings of modern detectors, we added warmup settings. lr_config = dict( - policy='step', - warmup='linear', + policy="step", + warmup="linear", warmup_iters=1000, warmup_ratio=1.0 / 1000, - step=[18, 24]) # the real step is [18*5, 24*5] + step=[18, 24], +) # the real step is [18*5, 24*5] runner = dict(max_epochs=28) # the real epoch is 28*5=140 # NOTE: `auto_scale_lr` is for automatically scaling LR, diff --git a/tools/analysis_tools/analyze_results.py b/tools/analysis_tools/analyze_results.py index 4d8b60c9..1fafb771 100644 --- a/tools/analysis_tools/analyze_results.py +++ b/tools/analysis_tools/analyze_results.py @@ -6,7 +6,6 @@ import mmcv import numpy as np from mmcv import Config, DictAction - from mmdet.core.evaluation import eval_map from mmdet.core.visualization import imshow_gt_det_bboxes from mmdet.datasets import build_dataset, get_loading_pipeline @@ -43,16 +42,17 @@ def bbox_map_eval(det_result, annotation, nproc=4): bbox_det_result = [det_result] # mAP iou_thrs = np.linspace( - .5, 0.95, int(np.round((0.95 - .5) / .05)) + 1, endpoint=True) + 0.5, 0.95, int(np.round((0.95 - 0.5) / 0.05)) + 1, endpoint=True + ) processes = [] workers = Pool(processes=nproc) for thr in iou_thrs: - p = workers.apply_async(eval_map, (bbox_det_result, [annotation]), { - 'iou_thr': thr, - 'logger': 'silent', - 'nproc': 1 - }) + p = workers.apply_async( + eval_map, + (bbox_det_result, [annotation]), + {"iou_thr": thr, "logger": "silent", "nproc": 1}, + ) processes.append(p) workers.close() @@ -80,21 +80,17 @@ class ResultVisualizer: with the prediction result. Default: False. """ - def __init__(self, - show=False, - wait_time=0, - score_thr=0, - overlay_gt_pred=False): + def __init__( + self, show=False, wait_time=0, score_thr=0, overlay_gt_pred=False + ): self.show = show self.wait_time = wait_time self.score_thr = score_thr self.overlay_gt_pred = overlay_gt_pred - def _save_image_gts_results(self, - dataset, - results, - performances, - out_dir=None): + def _save_image_gts_results( + self, dataset, results, performances, out_dir=None + ): """Display or save image with groung truths and predictions from a model. @@ -114,16 +110,16 @@ def _save_image_gts_results(self, data_info = dataset.prepare_train_img(index) # calc save file path - filename = data_info['filename'] - if data_info['img_prefix'] is not None: - filename = osp.join(data_info['img_prefix'], filename) + filename = data_info["filename"] + if data_info["img_prefix"] is not None: + filename = osp.join(data_info["img_prefix"], filename) else: - filename = data_info['filename'] + filename = data_info["filename"] fname, name = osp.splitext(osp.basename(filename)) - save_filename = fname + '_' + str(round(performance, 3)) + name + save_filename = fname + "_" + str(round(performance, 3)) + name out_file = osp.join(out_dir, save_filename) imshow_gt_det_bboxes( - data_info['img'], + data_info["img"], data_info, results[index], dataset.CLASSES, @@ -137,13 +133,10 @@ def _save_image_gts_results(self, score_thr=self.score_thr, wait_time=self.wait_time, out_file=out_file, - overlay_gt_pred=self.overlay_gt_pred) + overlay_gt_pred=self.overlay_gt_pred, + ) - def evaluate_and_show(self, - dataset, - results, - topk=20, - show_dir='work_dir'): + def evaluate_and_show(self, dataset, results, topk=20, show_dir="work_dir"): """Evaluate and show results. Args: @@ -163,21 +156,22 @@ def evaluate_and_show(self, if isinstance(results[0], dict): good_samples, bad_samples = self.panoptic_evaluate( - dataset, results, topk=topk) + dataset, results, topk=topk + ) elif isinstance(results[0], list): good_samples, bad_samples = self.detection_evaluate( - dataset, results, topk=topk) + dataset, results, topk=topk + ) elif isinstance(results[0], tuple): results_ = [result[0] for result in results] good_samples, bad_samples = self.detection_evaluate( - dataset, results_, topk=topk) + dataset, results_, topk=topk + ) else: - raise 'The format of result is not supported yet. ' \ - 'Current dict for panoptic segmentation and list ' \ - 'or tuple for object detection are supported.' + raise "The format of result is not supported yet. " "Current dict for panoptic segmentation and list " "or tuple for object detection are supported." - good_dir = osp.abspath(osp.join(show_dir, 'good')) - bad_dir = osp.abspath(osp.join(show_dir, 'bad')) + good_dir = osp.abspath(osp.join(show_dir, "good")) + bad_dir = osp.abspath(osp.join(show_dir, "bad")) self._save_image_gts_results(dataset, results, good_samples, good_dir) self._save_image_gts_results(dataset, results, bad_samples, bad_dir) @@ -208,11 +202,11 @@ def detection_evaluate(self, dataset, results, topk=20, eval_fn=None): prog_bar = mmcv.ProgressBar(len(results)) _mAPs = {} - for i, (result, ) in enumerate(zip(results)): + for i, (result,) in enumerate(zip(results)): # self.dataset[i] should not call directly # because there is a risk of mismatch data_info = dataset.prepare_train_img(i) - mAP = eval_fn(result, data_info['ann_info']) + mAP = eval_fn(result, data_info["ann_info"]) _mAPs[i] = mAP prog_bar.update() # descending select topk image @@ -245,31 +239,34 @@ def panoptic_evaluate(self, dataset, results, topk=20): gt_json = dataset.coco.img_ann_map result_files, tmp_dir = dataset.format_results(results) - pred_json = mmcv.load(result_files['panoptic'])['annotations'] - pred_folder = osp.join(tmp_dir.name, 'panoptic') + pred_json = mmcv.load(result_files["panoptic"])["annotations"] + pred_folder = osp.join(tmp_dir.name, "panoptic") gt_folder = dataset.seg_prefix pqs = {} prog_bar = mmcv.ProgressBar(len(results)) for i in range(len(results)): data_info = dataset.prepare_train_img(i) - image_id = data_info['img_info']['id'] + image_id = data_info["img_info"]["id"] gt_ann = { - 'image_id': image_id, - 'segments_info': gt_json[image_id], - 'file_name': data_info['img_info']['segm_file'] + "image_id": image_id, + "segments_info": gt_json[image_id], + "file_name": data_info["img_info"]["segm_file"], } pred_ann = pred_json[i] pq_stat = pq_compute_single_core( - i, [(gt_ann, pred_ann)], + i, + [(gt_ann, pred_ann)], gt_folder, pred_folder, dataset.categories, dataset.file_client, - print_log=False) + print_log=False, + ) pq_results, classwise_results = pq_stat.pq_average( - dataset.categories, isthing=None) - pqs[i] = pq_results['pq'] + dataset.categories, isthing=None + ) + pqs[i] = pq_results["pq"] prog_bar.update() if tmp_dir is not None: @@ -285,47 +282,55 @@ def panoptic_evaluate(self, dataset, results, topk=20): def parse_args(): parser = argparse.ArgumentParser( - description='MMDet eval image prediction result for each') - parser.add_argument('config', help='test config file path') + description="MMDet eval image prediction result for each" + ) + parser.add_argument("config", help="test config file path") parser.add_argument( - 'prediction_path', help='prediction path where test pkl result') + "prediction_path", help="prediction path where test pkl result" + ) parser.add_argument( - 'show_dir', help='directory where painted images will be saved') - parser.add_argument('--show', action='store_true', help='show results') + "show_dir", help="directory where painted images will be saved" + ) + parser.add_argument("--show", action="store_true", help="show results") parser.add_argument( - '--wait-time', + "--wait-time", type=float, default=0, - help='the interval of show (s), 0 is block') + help="the interval of show (s), 0 is block", + ) parser.add_argument( - '--topk', + "--topk", default=20, type=int, - help='saved Number of the highest topk ' - 'and lowest topk after index sorting') + help="saved Number of the highest topk " + "and lowest topk after index sorting", + ) parser.add_argument( - '--show-score-thr', + "--show-score-thr", type=float, default=0, - help='score threshold (default: 0.)') + help="score threshold (default: 0.)", + ) parser.add_argument( - '--overlay-gt-pred', - action='store_true', - help='whether to plot gts and predictions on the same image.' - 'If False, predictions and gts will be plotted on two same' - 'image which will be concatenated in vertical direction.' - 'The image above is drawn with gt, and the image below is' - 'drawn with the prediction result.') + "--overlay-gt-pred", + action="store_true", + help="whether to plot gts and predictions on the same image." + "If False, predictions and gts will be plotted on two same" + "image which will be concatenated in vertical direction." + "The image above is drawn with gt, and the image below is" + "drawn with the prediction result.", + ) parser.add_argument( - '--cfg-options', - nargs='+', + "--cfg-options", + nargs="+", action=DictAction, - help='override some settings in the used config, the key-value pair ' - 'in xxx=yyy format will be merged into config file. If the value to ' + help="override some settings in the used config, the key-value pair " + "in xxx=yyy format will be merged into config file. If the value to " 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' - 'Note that the quotation marks are necessary and that no white space ' - 'is allowed.') + "Note that the quotation marks are necessary and that no white space " + "is allowed.", + ) args = parser.parse_args() return args @@ -347,23 +352,29 @@ def main(): cfg.merge_from_dict(args.cfg_options) cfg.data.test.test_mode = True - cfg.data.test.pop('samples_per_gpu', 0) - if cfg.data.train.type in ('MultiImageMixDataset', 'ClassBalancedDataset', - 'RepeatDataset', 'ConcatDataset'): + cfg.data.test.pop("samples_per_gpu", 0) + if cfg.data.train.type in ( + "MultiImageMixDataset", + "ClassBalancedDataset", + "RepeatDataset", + "ConcatDataset", + ): cfg.data.test.pipeline = get_loading_pipeline( - cfg.data.train.dataset.pipeline) + cfg.data.train.dataset.pipeline + ) else: cfg.data.test.pipeline = get_loading_pipeline(cfg.data.train.pipeline) dataset = build_dataset(cfg.data.test) outputs = mmcv.load(args.prediction_path) - result_visualizer = ResultVisualizer(args.show, args.wait_time, - args.show_score_thr, - args.overlay_gt_pred) + result_visualizer = ResultVisualizer( + args.show, args.wait_time, args.show_score_thr, args.overlay_gt_pred + ) result_visualizer.evaluate_and_show( - dataset, outputs, topk=args.topk, show_dir=args.show_dir) + dataset, outputs, topk=args.topk, show_dir=args.show_dir + ) -if __name__ == '__main__': +if __name__ == "__main__": main() From 84d0a83884432f7039621372a47a50ccd173cbba Mon Sep 17 00:00:00 2001 From: ArchieAlexArkhipov Date: Sat, 6 May 2023 20:17:30 +0000 Subject: [PATCH 03/25] cycle centernet l1 --- mmdet/models/dense_heads/__init__.py | 62 ++- mmdet/models/dense_heads/centernet_head.py | 165 +++--- .../dense_heads/cycle_centernet_head.py | 497 ++++++++++++++++++ 3 files changed, 610 insertions(+), 114 deletions(-) create mode 100644 mmdet/models/dense_heads/cycle_centernet_head.py diff --git a/mmdet/models/dense_heads/__init__.py b/mmdet/models/dense_heads/__init__.py index 1c228699..725cfec7 100644 --- a/mmdet/models/dense_heads/__init__.py +++ b/mmdet/models/dense_heads/__init__.py @@ -41,18 +41,56 @@ from .yolo_head import YOLOV3Head from .yolof_head import YOLOFHead from .yolox_head import YOLOXHead +from .cycle_centernet_head import CycleCenterNetHeadL1 __all__ = [ - 'AnchorFreeHead', 'AnchorHead', 'GuidedAnchorHead', 'FeatureAdaption', - 'RPNHead', 'GARPNHead', 'RetinaHead', 'RetinaSepBNHead', 'GARetinaHead', - 'SSDHead', 'FCOSHead', 'RepPointsHead', 'FoveaHead', - 'FreeAnchorRetinaHead', 'ATSSHead', 'FSAFHead', 'NASFCOSHead', - 'PISARetinaHead', 'PISASSDHead', 'GFLHead', 'CornerHead', 'YOLACTHead', - 'YOLACTSegmHead', 'YOLACTProtonet', 'YOLOV3Head', 'PAAHead', - 'SABLRetinaHead', 'CentripetalHead', 'VFNetHead', 'StageCascadeRPNHead', - 'CascadeRPNHead', 'EmbeddingRPNHead', 'LDHead', 'AutoAssignHead', - 'DETRHead', 'YOLOFHead', 'DeformableDETRHead', 'SOLOHead', - 'DecoupledSOLOHead', 'CenterNetHead', 'YOLOXHead', - 'DecoupledSOLOLightHead', 'LADHead', 'TOODHead', 'MaskFormerHead', - 'Mask2FormerHead', 'SOLOV2Head', 'DDODHead' + "AnchorFreeHead", + "AnchorHead", + "GuidedAnchorHead", + "FeatureAdaption", + "RPNHead", + "GARPNHead", + "RetinaHead", + "RetinaSepBNHead", + "GARetinaHead", + "SSDHead", + "FCOSHead", + "RepPointsHead", + "FoveaHead", + "FreeAnchorRetinaHead", + "ATSSHead", + "FSAFHead", + "NASFCOSHead", + "PISARetinaHead", + "PISASSDHead", + "GFLHead", + "CornerHead", + "YOLACTHead", + "YOLACTSegmHead", + "YOLACTProtonet", + "YOLOV3Head", + "PAAHead", + "SABLRetinaHead", + "CentripetalHead", + "VFNetHead", + "StageCascadeRPNHead", + "CascadeRPNHead", + "EmbeddingRPNHead", + "LDHead", + "AutoAssignHead", + "DETRHead", + "YOLOFHead", + "DeformableDETRHead", + "SOLOHead", + "DecoupledSOLOHead", + "CenterNetHead", + "YOLOXHead", + "DecoupledSOLOLightHead", + "LADHead", + "TOODHead", + "MaskFormerHead", + "Mask2FormerHead", + "SOLOV2Head", + "DDODHead", + "CycleCenterNetHeadL1", ] diff --git a/mmdet/models/dense_heads/centernet_head.py b/mmdet/models/dense_heads/centernet_head.py index b9d5d2f0..26cde2e9 100644 --- a/mmdet/models/dense_heads/centernet_head.py +++ b/mmdet/models/dense_heads/centernet_head.py @@ -8,8 +8,7 @@ from mmdet.core import multi_apply from mmdet.models import HEADS, build_loss from mmdet.models.utils import gaussian_radius, gen_gaussian_target -from ..utils.gaussian_target import (get_local_maximum, get_topk_from_heatmap, - transpose_and_gather_feat) +from ..utils.gaussian_target import get_local_maximum, get_topk_from_heatmap, transpose_and_gather_feat from .base_dense_head import BaseDenseHead from .dense_test_mixins import BBoxTestMixin @@ -35,21 +34,21 @@ class CenterNetHead(BaseDenseHead, BBoxTestMixin): Default: None """ - def __init__(self, - in_channel, - feat_channel, - num_classes, - loss_center_heatmap=dict( - type='GaussianFocalLoss', loss_weight=1.0), - loss_wh=dict(type='L1Loss', loss_weight=0.1), - loss_offset=dict(type='L1Loss', loss_weight=1.0), - train_cfg=None, - test_cfg=None, - init_cfg=None): + def __init__( + self, + in_channel, + feat_channel, + num_classes, + loss_center_heatmap=dict(type="GaussianFocalLoss", loss_weight=1.0), + loss_wh=dict(type="L1Loss", loss_weight=0.1), + loss_offset=dict(type="L1Loss", loss_weight=1.0), + train_cfg=None, + test_cfg=None, + init_cfg=None, + ): super(CenterNetHead, self).__init__(init_cfg) self.num_classes = num_classes - self.heatmap_head = self._build_head(in_channel, feat_channel, - num_classes) + self.heatmap_head = self._build_head(in_channel, feat_channel, num_classes) self.wh_head = self._build_head(in_channel, feat_channel, 2) self.offset_head = self._build_head(in_channel, feat_channel, 2) @@ -66,7 +65,8 @@ def _build_head(self, in_channel, feat_channel, out_channel): layer = nn.Sequential( nn.Conv2d(in_channel, feat_channel, kernel_size=3, padding=1), nn.ReLU(inplace=True), - nn.Conv2d(feat_channel, out_channel, kernel_size=1)) + nn.Conv2d(feat_channel, out_channel, kernel_size=1), + ) return layer def init_weights(self): @@ -112,15 +112,10 @@ def forward_single(self, feat): offset_pred = self.offset_head(feat) return center_heatmap_pred, wh_pred, offset_pred - @force_fp32(apply_to=('center_heatmap_preds', 'wh_preds', 'offset_preds')) - def loss(self, - center_heatmap_preds, - wh_preds, - offset_preds, - gt_bboxes, - gt_labels, - img_metas, - gt_bboxes_ignore=None): + @force_fp32(apply_to=("center_heatmap_preds", "wh_preds", "offset_preds")) + def loss( + self, center_heatmap_preds, wh_preds, offset_preds, gt_bboxes, gt_labels, img_metas, gt_bboxes_ignore=None + ): """Compute losses of the head. Args: @@ -144,39 +139,30 @@ def loss(self, - loss_wh (Tensor): loss of hw heatmap - loss_offset (Tensor): loss of offset heatmap. """ - assert len(center_heatmap_preds) == len(wh_preds) == len( - offset_preds) == 1 + assert len(center_heatmap_preds) == len(wh_preds) == len(offset_preds) == 1 center_heatmap_pred = center_heatmap_preds[0] wh_pred = wh_preds[0] offset_pred = offset_preds[0] - target_result, avg_factor = self.get_targets(gt_bboxes, gt_labels, - center_heatmap_pred.shape, - img_metas[0]['pad_shape']) + target_result, avg_factor = self.get_targets( + gt_bboxes, gt_labels, center_heatmap_pred.shape, img_metas[0]["pad_shape"] + ) - center_heatmap_target = target_result['center_heatmap_target'] - wh_target = target_result['wh_target'] - offset_target = target_result['offset_target'] - wh_offset_target_weight = target_result['wh_offset_target_weight'] + center_heatmap_target = target_result["center_heatmap_target"] + wh_target = target_result["wh_target"] + offset_target = target_result["offset_target"] + wh_offset_target_weight = target_result["wh_offset_target_weight"] # Since the channel of wh_target and offset_target is 2, the avg_factor # of loss_center_heatmap is always 1/2 of loss_wh and loss_offset. loss_center_heatmap = self.loss_center_heatmap( - center_heatmap_pred, center_heatmap_target, avg_factor=avg_factor) - loss_wh = self.loss_wh( - wh_pred, - wh_target, - wh_offset_target_weight, - avg_factor=avg_factor * 2) + center_heatmap_pred, center_heatmap_target, avg_factor=avg_factor + ) + loss_wh = self.loss_wh(wh_pred, wh_target, wh_offset_target_weight, avg_factor=avg_factor * 2) loss_offset = self.loss_offset( - offset_pred, - offset_target, - wh_offset_target_weight, - avg_factor=avg_factor * 2) - return dict( - loss_center_heatmap=loss_center_heatmap, - loss_wh=loss_wh, - loss_offset=loss_offset) + offset_pred, offset_target, wh_offset_target_weight, avg_factor=avg_factor * 2 + ) + return dict(loss_center_heatmap=loss_center_heatmap, loss_wh=loss_wh, loss_offset=loss_offset) def get_targets(self, gt_bboxes, gt_labels, feat_shape, img_shape): """Compute regression and classification targets in multiple images. @@ -206,12 +192,10 @@ def get_targets(self, gt_bboxes, gt_labels, feat_shape, img_shape): width_ratio = float(feat_w / img_w) height_ratio = float(feat_h / img_h) - center_heatmap_target = gt_bboxes[-1].new_zeros( - [bs, self.num_classes, feat_h, feat_w]) + center_heatmap_target = gt_bboxes[-1].new_zeros([bs, self.num_classes, feat_h, feat_w]) wh_target = gt_bboxes[-1].new_zeros([bs, 2, feat_h, feat_w]) offset_target = gt_bboxes[-1].new_zeros([bs, 2, feat_h, feat_w]) - wh_offset_target_weight = gt_bboxes[-1].new_zeros( - [bs, 2, feat_h, feat_w]) + wh_offset_target_weight = gt_bboxes[-1].new_zeros([bs, 2, feat_h, feat_w]) for batch_id in range(bs): gt_bbox = gt_bboxes[batch_id] @@ -225,12 +209,10 @@ def get_targets(self, gt_bboxes, gt_labels, feat_shape, img_shape): ctx, cty = ct scale_box_h = (gt_bbox[j][3] - gt_bbox[j][1]) * height_ratio scale_box_w = (gt_bbox[j][2] - gt_bbox[j][0]) * width_ratio - radius = gaussian_radius([scale_box_h, scale_box_w], - min_overlap=0.3) + radius = gaussian_radius([scale_box_h, scale_box_w], min_overlap=0.3) radius = max(0, int(radius)) ind = gt_label[j] - gen_gaussian_target(center_heatmap_target[batch_id, ind], - [ctx_int, cty_int], radius) + gen_gaussian_target(center_heatmap_target[batch_id, ind], [ctx_int, cty_int], radius) wh_target[batch_id, 0, cty_int, ctx_int] = scale_box_w wh_target[batch_id, 1, cty_int, ctx_int] = scale_box_h @@ -245,17 +227,12 @@ def get_targets(self, gt_bboxes, gt_labels, feat_shape, img_shape): center_heatmap_target=center_heatmap_target, wh_target=wh_target, offset_target=offset_target, - wh_offset_target_weight=wh_offset_target_weight) + wh_offset_target_weight=wh_offset_target_weight, + ) return target_result, avg_factor - @force_fp32(apply_to=('center_heatmap_preds', 'wh_preds', 'offset_preds')) - def get_bboxes(self, - center_heatmap_preds, - wh_preds, - offset_preds, - img_metas, - rescale=True, - with_nms=False): + @force_fp32(apply_to=("center_heatmap_preds", "wh_preds", "offset_preds")) + def get_bboxes(self, center_heatmap_preds, wh_preds, offset_preds, img_metas, rescale=True, with_nms=False): """Transform network output for a batch into bbox predictions. Args: @@ -280,27 +257,24 @@ def get_bboxes(self, each element represents the class label of the corresponding box. """ - assert len(center_heatmap_preds) == len(wh_preds) == len( - offset_preds) == 1 + assert len(center_heatmap_preds) == len(wh_preds) == len(offset_preds) == 1 result_list = [] for img_id in range(len(img_metas)): result_list.append( self._get_bboxes_single( - center_heatmap_preds[0][img_id:img_id + 1, ...], - wh_preds[0][img_id:img_id + 1, ...], - offset_preds[0][img_id:img_id + 1, ...], + center_heatmap_preds[0][img_id : img_id + 1, ...], + wh_preds[0][img_id : img_id + 1, ...], + offset_preds[0][img_id : img_id + 1, ...], img_metas[img_id], rescale=rescale, - with_nms=with_nms)) + with_nms=with_nms, + ) + ) return result_list - def _get_bboxes_single(self, - center_heatmap_pred, - wh_pred, - offset_pred, - img_meta, - rescale=False, - with_nms=True): + def _get_bboxes_single( + self, center_heatmap_pred, wh_pred, offset_pred, img_meta, rescale=False, with_nms=True + ): """Transform outputs of a single image into bbox results. Args: @@ -328,33 +302,25 @@ def _get_bboxes_single(self, center_heatmap_pred, wh_pred, offset_pred, - img_meta['batch_input_shape'], + img_meta["batch_input_shape"], k=self.test_cfg.topk, - kernel=self.test_cfg.local_maximum_kernel) + kernel=self.test_cfg.local_maximum_kernel, + ) det_bboxes = batch_det_bboxes.view([-1, 5]) det_labels = batch_labels.view(-1) - batch_border = det_bboxes.new_tensor(img_meta['border'])[..., - [2, 0, 2, 0]] + batch_border = det_bboxes.new_tensor(img_meta["border"])[..., [2, 0, 2, 0]] det_bboxes[..., :4] -= batch_border if rescale: - det_bboxes[..., :4] /= det_bboxes.new_tensor( - img_meta['scale_factor']) + det_bboxes[..., :4] /= det_bboxes.new_tensor(img_meta["scale_factor"]) if with_nms: - det_bboxes, det_labels = self._bboxes_nms(det_bboxes, det_labels, - self.test_cfg) + det_bboxes, det_labels = self._bboxes_nms(det_bboxes, det_labels, self.test_cfg) return det_bboxes, det_labels - def decode_heatmap(self, - center_heatmap_pred, - wh_pred, - offset_pred, - img_shape, - k=100, - kernel=3): + def decode_heatmap(self, center_heatmap_pred, wh_pred, offset_pred, img_shape, k=100, kernel=3): """Transform outputs into detections raw bbox prediction. Args: @@ -378,11 +344,9 @@ def decode_heatmap(self, height, width = center_heatmap_pred.shape[2:] inp_h, inp_w = img_shape - center_heatmap_pred = get_local_maximum( - center_heatmap_pred, kernel=kernel) + center_heatmap_pred = get_local_maximum(center_heatmap_pred, kernel=kernel) - *batch_dets, topk_ys, topk_xs = get_topk_from_heatmap( - center_heatmap_pred, k=k) + *batch_dets, topk_ys, topk_xs = get_topk_from_heatmap(center_heatmap_pred, k=k) batch_scores, batch_index, batch_topk_labels = batch_dets wh = transpose_and_gather_feat(wh_pred, batch_index) @@ -395,16 +359,13 @@ def decode_heatmap(self, br_y = (topk_ys + wh[..., 1] / 2) * (inp_h / height) batch_bboxes = torch.stack([tl_x, tl_y, br_x, br_y], dim=2) - batch_bboxes = torch.cat((batch_bboxes, batch_scores[..., None]), - dim=-1) + batch_bboxes = torch.cat((batch_bboxes, batch_scores[..., None]), dim=-1) return batch_bboxes, batch_topk_labels def _bboxes_nms(self, bboxes, labels, cfg): if labels.numel() > 0: max_num = cfg.max_per_img - bboxes, keep = batched_nms(bboxes[:, :4], bboxes[:, - -1].contiguous(), - labels, cfg.nms) + bboxes, keep = batched_nms(bboxes[:, :4], bboxes[:, -1].contiguous(), labels, cfg.nms) if max_num > 0: bboxes = bboxes[:max_num] labels = labels[keep][:max_num] diff --git a/mmdet/models/dense_heads/cycle_centernet_head.py b/mmdet/models/dense_heads/cycle_centernet_head.py new file mode 100644 index 00000000..33bb0015 --- /dev/null +++ b/mmdet/models/dense_heads/cycle_centernet_head.py @@ -0,0 +1,497 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +import torch.nn as nn +from mmcv.cnn import bias_init_with_prob, normal_init +from mmcv.ops import batched_nms +from mmcv.runner import force_fp32 + +from mmdet.core import multi_apply +from mmdet.models import HEADS, build_loss +from mmdet.models.utils import gaussian_radius, gen_gaussian_target +from ..utils.gaussian_target import get_local_maximum, get_topk_from_heatmap, transpose_and_gather_feat +from .base_dense_head import BaseDenseHead +from .dense_test_mixins import BBoxTestMixin + + +@HEADS.register_module() +class CycleCenterNetHeadL1(BaseDenseHead, BBoxTestMixin): + """Parsing Table Structures in the Wild Head. CycleCenterHead use center_point to indicate object's + position. Paper link + + Args: + in_channel (int): Number of channel in the input feature map. + feat_channel (int): Number of channel in the intermediate feature map. + num_classes (int): Number of categories excluding the background + category. + loss_center_heatmap (dict | None): Config of center heatmap loss. + Default: GaussianFocalLoss. + loss_wh (dict | None): Config of wh loss. Default: L1Loss. + loss_offset (dict | None): Config of offset loss. Default: L1Loss. + train_cfg (dict | None): Training config. Useless in CenterNet, + but we keep this variable for SingleStageDetector. Default: None. + test_cfg (dict | None): Testing config of CenterNet. Default: None. + init_cfg (dict or list[dict], optional): Initialization config dict. + Default: None + """ + + def __init__( + self, + in_channel, + feat_channel, + num_classes, + loss_center_heatmap=dict(type="GaussianFocalLoss", loss_weight=1.0), + loss_wh=dict(type="L1Loss", loss_weight=0.1), + loss_offset=dict(type="L1Loss", loss_weight=1.0), + loss_c2v=dict(type="L1Loss", loss_weight=1.0), + loss_v2c=dict(type="L1Loss", loss_weight=0.5), + # loss_pairing=dict(type="PairingLoss", loss_weight=1.0), + train_cfg=None, + test_cfg=None, + init_cfg=None, + ): + super(CycleCenterNetHeadL1, self).__init__(init_cfg) + self.num_classes = num_classes + self.heatmap_head = self._build_head(in_channel, feat_channel, num_classes) + self.wh_head = self._build_head(in_channel, feat_channel, 2) + self.offset_head = self._build_head(in_channel, feat_channel, 2) + self.center2vertex_head = self._build_head(in_channel, feat_channel, 8) + self.vertex2center_head = self._build_head(in_channel, feat_channel, 8) + + self.loss_center_heatmap = build_loss(loss_center_heatmap) + self.loss_wh = build_loss(loss_wh) + self.loss_offset = build_loss(loss_offset) + # self.loss_pairing = build_loss(loss_pairing) + self.loss_c2v = build_loss(loss_c2v) + self.loss_v2c = build_loss(loss_v2c) + + self.train_cfg = train_cfg + self.test_cfg = test_cfg + self.fp16_enabled = False + + def _build_head(self, in_channel, feat_channel, out_channel): + """Build head for each branch.""" + layer = nn.Sequential( + nn.Conv2d(in_channel, feat_channel, kernel_size=3, padding=1), + nn.ReLU(inplace=True), + nn.Conv2d(feat_channel, out_channel, kernel_size=1), + ) + return layer + + def init_weights(self): + """Initialize weights of the head.""" + bias_init = bias_init_with_prob(0.1) + self.heatmap_head[-1].bias.data.fill_(bias_init) + for head in [self.wh_head, self.offset_head, self.center2vertex_head, self.vertex2center_head]: + for m in head.modules(): + if isinstance(m, nn.Conv2d): + normal_init(m, std=0.001) + + def forward(self, feats): + """Forward features. Notice CenterNet head does not use FPN. + + Args: + feats (tuple[Tensor]): Features from the upstream network, each is + a 4D-tensor. + + Returns: + center_heatmap_preds (List[Tensor]): center predict heatmaps for + all levels, the channels number is num_classes. + wh_preds (List[Tensor]): wh predicts for all levels, the channels + number is 2. + offset_preds (List[Tensor]): offset predicts for all levels, the + channels number is 2. + """ + return multi_apply(self.forward_single, feats) + + def forward_single(self, feat): + """Forward feature of a single level. + + Args: + feat (Tensor): Feature of a single level. + + Returns: + center_heatmap_pred (Tensor): center predict heatmaps, the + channels number is num_classes. + wh_pred (Tensor): wh predicts, the channels number is 2. + offset_pred (Tensor): offset predicts, the channels number is 2. + """ + center_heatmap_pred = self.heatmap_head(feat).sigmoid() + wh_pred = self.wh_head(feat) + offset_pred = self.offset_head(feat) + center2vertex_pred = self.center2vertex_head(feat) + vertex2center_pred = self.vertex2center_head(feat) + return center_heatmap_pred, wh_pred, offset_pred, center2vertex_pred, vertex2center_pred + + @force_fp32( + apply_to=("center_heatmap_preds", "wh_preds", "offset_preds", "center2vertex_pred", "vertex2center_pred") + ) + def loss( + self, + center_heatmap_preds, + wh_preds, + offset_preds, + center2vertex_pred, + vertex2center_pred, + gt_bboxes, + gt_labels, + img_metas, + gt_bboxes_ignore=None, + ): + """Compute losses of the head. + + Args: + center_heatmap_preds (list[Tensor]): center predict heatmaps for + all levels with shape (B, num_classes, H, W). + wh_preds (list[Tensor]): wh predicts for all levels with + shape (B, 2, H, W). + offset_preds (list[Tensor]): offset predicts for all levels + with shape (B, 2, H, W). + center2vertex_pred (list[Tensor]): center2vertex predicts for all levels + with shape (B, 8, H, W). + vertex2center_pred (list[Tensor]): vertex2center predicts for all levels + with shape (B, 8, H, W). + gt_bboxes (list[Tensor]): Ground truth bboxes for each image with + shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + gt_labels (list[Tensor]): class indices corresponding to each box. + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + gt_bboxes_ignore (None | list[Tensor]): specify which bounding + boxes can be ignored when computing the loss. Default: None + + Returns: + dict[str, Tensor]: which has components below: + - loss_center_heatmap (Tensor): loss of center heatmap. + - loss_wh (Tensor): loss of hw heatmap + - loss_offset (Tensor): loss of offset heatmap. + """ + assert ( + len(center_heatmap_preds) + == len(wh_preds) + == len(offset_preds) + == len(center2vertex_pred) + == len(vertex2center_pred) + == 1 + ) + center_heatmap_pred = center_heatmap_preds[0] + wh_pred = wh_preds[0] + offset_pred = offset_preds[0] + center2vertex_pred = center2vertex_pred[0] + vertex2center_pred = vertex2center_pred[0] + + target_result, avg_factor = self.get_targets( + gt_bboxes, gt_labels, center_heatmap_pred.shape, img_metas[0]["pad_shape"] + ) + + center_heatmap_target = target_result["center_heatmap_target"] + wh_target = target_result["wh_target"] + offset_target = target_result["offset_target"] + wh_offset_target_weight = target_result["wh_offset_target_weight"] + center2vertex_target = target_result["center2vertex_target"] + vertex2center_target = target_result["vertex2center_target"] + c2v_v2c_target_weight = target_result["c2v_v2c_target_weight"] + + # Since the channel of wh_target and offset_target is 2, the avg_factor + # of loss_center_heatmap is always 1/2 of loss_wh and loss_offset. + loss_center_heatmap = self.loss_center_heatmap( + center_heatmap_pred, center_heatmap_target, avg_factor=avg_factor + ) + loss_wh = self.loss_wh(wh_pred, wh_target, wh_offset_target_weight, avg_factor=avg_factor * 2) + loss_offset = self.loss_offset( + offset_pred, offset_target, wh_offset_target_weight, avg_factor=avg_factor * 2 + ) + # loss_pairing = self.loss_pairing( + # center2vertex_pred, + # vertex2center_pred, + # center2vertex_target, + # vertex2center_target, + # ..., + # avg_factor=avg_factor, + # ) + loss_c2v = self.loss_c2v( + center2vertex_pred, center2vertex_target, c2v_v2c_target_weight, avg_factor=avg_factor * 8 + ) + loss_v2c = self.loss_v2c( + vertex2center_pred, vertex2center_target, c2v_v2c_target_weight, avg_factor=avg_factor * 8 + ) + + return dict( + loss_center_heatmap=loss_center_heatmap, + loss_wh=loss_wh, + loss_offset=loss_offset, + # loss_pairing=loss_pairing, + loss_c2v=loss_c2v, + loss_v2c=loss_v2c, + ) + + def get_targets(self, gt_bboxes, gt_labels, feat_shape, img_shape): + """Compute regression and classification targets in multiple images. + + Args: + gt_bboxes (list[Tensor]): Ground truth bboxes for each image with + shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. + gt_labels (list[Tensor]): class indices corresponding to each box. + feat_shape (list[int]): feature map shape with value [B, _, H, W] + img_shape (list[int]): image shape in [h, w] format. + + Returns: + tuple[dict,float]: The float value is mean avg_factor, the dict has + components below: + - center_heatmap_target (Tensor): targets of center heatmap, \ + shape (B, num_classes, H, W). + - wh_target (Tensor): targets of wh predict, shape \ + (B, 2, H, W). + - offset_target (Tensor): targets of offset predict, shape \ + (B, 2, H, W). + - wh_offset_target_weight (Tensor): weights of wh and offset \ + predict, shape (B, 2, H, W). + - center2vertex_target (Tensor): targets of center2vertex predict, shape \ + (B, 8, H, W). + - vertex2center_target (Tensor): targets of vertex2center predict, shape \ + (B, 8, H, W). + """ + img_h, img_w = img_shape[:2] + bs, _, feat_h, feat_w = feat_shape + + width_ratio = float(feat_w / img_w) + height_ratio = float(feat_h / img_h) + + center_heatmap_target = gt_bboxes[-1].new_zeros([bs, self.num_classes, feat_h, feat_w]) + wh_target = gt_bboxes[-1].new_zeros([bs, 2, feat_h, feat_w]) + offset_target = gt_bboxes[-1].new_zeros([bs, 2, feat_h, feat_w]) + wh_offset_target_weight = gt_bboxes[-1].new_zeros([bs, 2, feat_h, feat_w]) + center2vertex_target = gt_bboxes[-1].new_zeros([bs, 8, feat_h, feat_w]) + vertex2center_target = gt_bboxes[-1].new_zeros([bs, 8, feat_h, feat_w]) + c2v_v2c_target_weight = gt_bboxes[-1].new_zeros([bs, 8, feat_h, feat_w]) + + for batch_id in range(bs): + gt_bbox = gt_bboxes[batch_id] + gt_label = gt_labels[batch_id] + center_x = (gt_bbox[:, [0]] + gt_bbox[:, [2]]) * width_ratio / 2 + center_y = (gt_bbox[:, [1]] + gt_bbox[:, [3]]) * height_ratio / 2 + gt_centers = torch.cat((center_x, center_y), dim=1) + + for j, ct in enumerate(gt_centers): + ctx_int, cty_int = ct.int() + ctx, cty = ct + scale_box_h = (gt_bbox[j][3] - gt_bbox[j][1]) * height_ratio + scale_box_w = (gt_bbox[j][2] - gt_bbox[j][0]) * width_ratio + radius = gaussian_radius([scale_box_h, scale_box_w], min_overlap=0.3) + radius = max(0, int(radius)) + ind = gt_label[j] + gen_gaussian_target(center_heatmap_target[batch_id, ind], [ctx_int, cty_int], radius) + + wh_target[batch_id, 0, cty_int, ctx_int] = scale_box_w + wh_target[batch_id, 1, cty_int, ctx_int] = scale_box_h + + offset_target[batch_id, 0, cty_int, ctx_int] = ctx - ctx_int + offset_target[batch_id, 1, cty_int, ctx_int] = cty - cty_int + + wh_offset_target_weight[batch_id, :, cty_int, ctx_int] = 1 + + center2vertex_target[batch_id, 0, cty_int, ctx_int] = -scale_box_w / 2 + center2vertex_target[batch_id, 1, cty_int, ctx_int] = -scale_box_h / 2 + center2vertex_target[batch_id, 2, cty_int, ctx_int] = scale_box_w / 2 + center2vertex_target[batch_id, 3, cty_int, ctx_int] = -scale_box_h / 2 + center2vertex_target[batch_id, 4, cty_int, ctx_int] = scale_box_w / 2 + center2vertex_target[batch_id, 5, cty_int, ctx_int] = scale_box_h / 2 + center2vertex_target[batch_id, 6, cty_int, ctx_int] = -scale_box_w / 2 + center2vertex_target[batch_id, 7, cty_int, ctx_int] = scale_box_h / 2 + + vertex2center_target[batch_id, 0, cty_int, ctx_int] = scale_box_w / 2 + vertex2center_target[batch_id, 1, cty_int, ctx_int] = scale_box_h / 2 + vertex2center_target[batch_id, 2, cty_int, ctx_int] = -scale_box_w / 2 + vertex2center_target[batch_id, 3, cty_int, ctx_int] = scale_box_h / 2 + vertex2center_target[batch_id, 4, cty_int, ctx_int] = -scale_box_w / 2 + vertex2center_target[batch_id, 5, cty_int, ctx_int] = -scale_box_h / 2 + vertex2center_target[batch_id, 6, cty_int, ctx_int] = scale_box_w / 2 + vertex2center_target[batch_id, 7, cty_int, ctx_int] = -scale_box_h / 2 + + c2v_v2c_target_weight[batch_id, :, cty_int, ctx_int] = 1 + avg_factor = max(1, center_heatmap_target.eq(1).sum()) + target_result = dict( + center_heatmap_target=center_heatmap_target, + wh_target=wh_target, + offset_target=offset_target, + wh_offset_target_weight=wh_offset_target_weight, + center2vertex_target=center2vertex_target, + vertex2center_target=vertex2center_target, + c2v_v2c_target_weight=c2v_v2c_target_weight, + ) + return target_result, avg_factor + + @force_fp32( + apply_to=("center_heatmap_preds", "wh_preds", "offset_preds", "center2vertex_preds", "vertex2center_preds") + ) + def get_bboxes( + self, + center_heatmap_preds, + wh_preds, + offset_preds, + center2vertex_preds, + vertex2center_preds, + img_metas, + rescale=True, + with_nms=False, + ): + """Transform network output for a batch into bbox predictions. + + Args: + center_heatmap_preds (list[Tensor]): Center predict heatmaps for + all levels with shape (B, num_classes, H, W). + wh_preds (list[Tensor]): WH predicts for all levels with + shape (B, 2, H, W). + offset_preds (list[Tensor]): Offset predicts for all levels + with shape (B, 2, H, W). + center2vertex_pred (list[Tensor]): center2vertex predicts for all levels + with shape (B, 8, H, W). + vertex2center_pred (list[Tensor]): vertex2center predicts for all levels + with shape (B, 8, H, W). + img_metas (list[dict]): Meta information of each image, e.g., + image size, scaling factor, etc. + rescale (bool): If True, return boxes in original image space. + Default: True. + with_nms (bool): If True, do nms before return boxes. + Default: False. + + Returns: + list[tuple[Tensor, Tensor]]: Each item in result_list is 2-tuple. + The first item is an (n, 5) tensor, where 5 represent + (tl_x, tl_y, br_x, br_y, score) and the score between 0 and 1. + The shape of the second tensor in the tuple is (n,), and + each element represents the class label of the corresponding + box. + """ + assert ( + len(center_heatmap_preds) + == len(wh_preds) + == len(offset_preds) + == len(center2vertex_preds) + == len(vertex2center_preds) + == 1 + ) + diff = (center2vertex_preds[0] - vertex2center_preds[0]) / 2 + wh_preds[0][:, 0, ...] = (-diff[:, 0, ...] + diff[:, 2, ...] + diff[:, 4, ...] - diff[:, 6, ...]) / 2 + wh_preds[0][:, 1, ...] = (-diff[:, 1, ...] - diff[:, 3, ...] + diff[:, 5, ...] - diff[:, 7, ...]) / 2 + result_list = [] + for img_id in range(len(img_metas)): + result_list.append( + self._get_bboxes_single( + center_heatmap_preds[0][img_id : img_id + 1, ...], + wh_preds[0][img_id : img_id + 1, ...], + offset_preds[0][img_id : img_id + 1, ...], + img_metas[img_id], + rescale=rescale, + with_nms=with_nms, + ) + ) + return result_list + + def _get_bboxes_single( + self, + center_heatmap_pred, + wh_pred, + offset_pred, + img_meta, + rescale=False, + with_nms=True, + ): + """Transform outputs of a single image into bbox results. + + Args: + center_heatmap_pred (Tensor): Center heatmap for current level with + shape (1, num_classes, H, W). + wh_pred (Tensor): WH heatmap for current level with shape + (1, num_classes, H, W). + offset_pred (Tensor): Offset for current level with shape + (1, corner_offset_channels, H, W). + # center2vertex_pred (list[Tensor]): center2vertex predicts for all levels + # with shape (1, 8, H, W). + # vertex2center_pred (list[Tensor]): vertex2center predicts for all levels + # with shape (1, 8, H, W). + img_meta (dict): Meta information of current image, e.g., + image size, scaling factor, etc. + rescale (bool): If True, return boxes in original image space. + Default: False. + with_nms (bool): If True, do nms before return boxes. + Default: True. + + Returns: + tuple[Tensor, Tensor]: The first item is an (n, 5) tensor, where + 5 represent (tl_x, tl_y, br_x, br_y, score) and the score + between 0 and 1. The shape of the second tensor in the tuple + is (n,), and each element represents the class label of the + corresponding box. + """ + batch_det_bboxes, batch_labels = self.decode_heatmap( + center_heatmap_pred, + wh_pred, + offset_pred, + img_meta["batch_input_shape"], + k=self.test_cfg.topk, + kernel=self.test_cfg.local_maximum_kernel, + ) + + det_bboxes = batch_det_bboxes.view([-1, 5]) + det_labels = batch_labels.view(-1) + + batch_border = det_bboxes.new_tensor(img_meta["border"])[..., [2, 0, 2, 0]] + det_bboxes[..., :4] -= batch_border + + if rescale: + det_bboxes[..., :4] /= det_bboxes.new_tensor(img_meta["scale_factor"]) + + if with_nms: + det_bboxes, det_labels = self._bboxes_nms(det_bboxes, det_labels, self.test_cfg) + return det_bboxes, det_labels + + def decode_heatmap(self, center_heatmap_pred, wh_pred, offset_pred, img_shape, k=100, kernel=3): + """Transform outputs into detections raw bbox prediction. + + Args: + center_heatmap_pred (Tensor): center predict heatmap, + shape (B, num_classes, H, W). + wh_pred (Tensor): wh predict, shape (B, 2, H, W). + offset_pred (Tensor): offset predict, shape (B, 2, H, W). + img_shape (list[int]): image shape in [h, w] format. + k (int): Get top k center keypoints from heatmap. Default 100. + kernel (int): Max pooling kernel for extract local maximum pixels. + Default 3. + + Returns: + tuple[torch.Tensor]: Decoded output of CenterNetHead, containing + the following Tensors: + + - batch_bboxes (Tensor): Coords of each box with shape (B, k, 5) + - batch_topk_labels (Tensor): Categories of each box with \ + shape (B, k) + """ + height, width = center_heatmap_pred.shape[2:] + inp_h, inp_w = img_shape + + center_heatmap_pred = get_local_maximum(center_heatmap_pred, kernel=kernel) + + *batch_dets, topk_ys, topk_xs = get_topk_from_heatmap(center_heatmap_pred, k=k) + batch_scores, batch_index, batch_topk_labels = batch_dets + + wh = transpose_and_gather_feat(wh_pred, batch_index) + offset = transpose_and_gather_feat(offset_pred, batch_index) + topk_xs = topk_xs + offset[..., 0] + topk_ys = topk_ys + offset[..., 1] + tl_x = (topk_xs - wh[..., 0] / 2) * (inp_w / width) + tl_y = (topk_ys - wh[..., 1] / 2) * (inp_h / height) + br_x = (topk_xs + wh[..., 0] / 2) * (inp_w / width) + br_y = (topk_ys + wh[..., 1] / 2) * (inp_h / height) + + batch_bboxes = torch.stack([tl_x, tl_y, br_x, br_y], dim=2) + batch_bboxes = torch.cat((batch_bboxes, batch_scores[..., None]), dim=-1) + return batch_bboxes, batch_topk_labels + + def _bboxes_nms(self, bboxes, labels, cfg): + if labels.numel() > 0: + max_num = cfg.max_per_img + bboxes, keep = batched_nms(bboxes[:, :4], bboxes[:, -1].contiguous(), labels, cfg.nms) + if max_num > 0: + bboxes = bboxes[:max_num] + labels = labels[keep][:max_num] + + return bboxes, labels From 909f0ca8ffe829a9e0447db56b453b4fe0eac12b Mon Sep 17 00:00:00 2001 From: ArchieAlexArkhipov Date: Sun, 7 May 2023 13:52:54 +0000 Subject: [PATCH 04/25] beta 2 heatmaps --- .../dense_heads/cycle_centernet_head.py | 293 ++++++++++++++---- 1 file changed, 228 insertions(+), 65 deletions(-) diff --git a/mmdet/models/dense_heads/cycle_centernet_head.py b/mmdet/models/dense_heads/cycle_centernet_head.py index 33bb0015..14b7e083 100644 --- a/mmdet/models/dense_heads/cycle_centernet_head.py +++ b/mmdet/models/dense_heads/cycle_centernet_head.py @@ -8,15 +8,20 @@ from mmdet.core import multi_apply from mmdet.models import HEADS, build_loss from mmdet.models.utils import gaussian_radius, gen_gaussian_target -from ..utils.gaussian_target import get_local_maximum, get_topk_from_heatmap, transpose_and_gather_feat +from ..utils.gaussian_target import ( + get_local_maximum, + get_topk_from_heatmap, + transpose_and_gather_feat, +) from .base_dense_head import BaseDenseHead from .dense_test_mixins import BBoxTestMixin @HEADS.register_module() class CycleCenterNetHeadL1(BaseDenseHead, BBoxTestMixin): - """Parsing Table Structures in the Wild Head. CycleCenterHead use center_point to indicate object's - position. Paper link + """Parsing Table Structures in the Wild Head. CycleCenterHead use + center_point to indicate object's position. + Paper link Args: in_channel (int): Number of channel in the input feature map. @@ -40,7 +45,7 @@ def __init__( feat_channel, num_classes, loss_center_heatmap=dict(type="GaussianFocalLoss", loss_weight=1.0), - loss_wh=dict(type="L1Loss", loss_weight=0.1), + # loss_wh=dict(type="L1Loss", loss_weight=0.1), loss_offset=dict(type="L1Loss", loss_weight=1.0), loss_c2v=dict(type="L1Loss", loss_weight=1.0), loss_v2c=dict(type="L1Loss", loss_weight=0.5), @@ -51,14 +56,16 @@ def __init__( ): super(CycleCenterNetHeadL1, self).__init__(init_cfg) self.num_classes = num_classes - self.heatmap_head = self._build_head(in_channel, feat_channel, num_classes) - self.wh_head = self._build_head(in_channel, feat_channel, 2) + self.heatmap_head = self._build_head( + in_channel, feat_channel, 2 * num_classes + ) + # self.wh_head = self._build_head(in_channel, feat_channel, 2) self.offset_head = self._build_head(in_channel, feat_channel, 2) self.center2vertex_head = self._build_head(in_channel, feat_channel, 8) self.vertex2center_head = self._build_head(in_channel, feat_channel, 8) self.loss_center_heatmap = build_loss(loss_center_heatmap) - self.loss_wh = build_loss(loss_wh) + # self.loss_wh = build_loss(loss_wh) self.loss_offset = build_loss(loss_offset) # self.loss_pairing = build_loss(loss_pairing) self.loss_c2v = build_loss(loss_c2v) @@ -81,7 +88,12 @@ def init_weights(self): """Initialize weights of the head.""" bias_init = bias_init_with_prob(0.1) self.heatmap_head[-1].bias.data.fill_(bias_init) - for head in [self.wh_head, self.offset_head, self.center2vertex_head, self.vertex2center_head]: + for head in [ + # self.wh_head, + self.offset_head, + self.center2vertex_head, + self.vertex2center_head, + ]: for m in head.modules(): if isinstance(m, nn.Conv2d): normal_init(m, std=0.001) @@ -116,19 +128,31 @@ def forward_single(self, feat): offset_pred (Tensor): offset predicts, the channels number is 2. """ center_heatmap_pred = self.heatmap_head(feat).sigmoid() - wh_pred = self.wh_head(feat) + # wh_pred = self.wh_head(feat) offset_pred = self.offset_head(feat) center2vertex_pred = self.center2vertex_head(feat) vertex2center_pred = self.vertex2center_head(feat) - return center_heatmap_pred, wh_pred, offset_pred, center2vertex_pred, vertex2center_pred + return ( + center_heatmap_pred, + # wh_pred, + offset_pred, + center2vertex_pred, + vertex2center_pred, + ) @force_fp32( - apply_to=("center_heatmap_preds", "wh_preds", "offset_preds", "center2vertex_pred", "vertex2center_pred") + apply_to=( + "center_heatmap_preds", + # "wh_preds", + "offset_preds", + "center2vertex_pred", + "vertex2center_pred", + ) ) def loss( self, center_heatmap_preds, - wh_preds, + # wh_preds, offset_preds, center2vertex_pred, vertex2center_pred, @@ -166,24 +190,27 @@ def loss( """ assert ( len(center_heatmap_preds) - == len(wh_preds) + # == len(wh_preds) == len(offset_preds) == len(center2vertex_pred) == len(vertex2center_pred) == 1 ) center_heatmap_pred = center_heatmap_preds[0] - wh_pred = wh_preds[0] + # wh_pred = wh_preds[0] offset_pred = offset_preds[0] center2vertex_pred = center2vertex_pred[0] vertex2center_pred = vertex2center_pred[0] target_result, avg_factor = self.get_targets( - gt_bboxes, gt_labels, center_heatmap_pred.shape, img_metas[0]["pad_shape"] + gt_bboxes, + gt_labels, + center_heatmap_pred.shape, + img_metas[0]["pad_shape"], ) center_heatmap_target = target_result["center_heatmap_target"] - wh_target = target_result["wh_target"] + # wh_target = target_result["wh_target"] offset_target = target_result["offset_target"] wh_offset_target_weight = target_result["wh_offset_target_weight"] center2vertex_target = target_result["center2vertex_target"] @@ -195,9 +222,17 @@ def loss( loss_center_heatmap = self.loss_center_heatmap( center_heatmap_pred, center_heatmap_target, avg_factor=avg_factor ) - loss_wh = self.loss_wh(wh_pred, wh_target, wh_offset_target_weight, avg_factor=avg_factor * 2) + # loss_wh = self.loss_wh( + # wh_pred, + # wh_target, + # wh_offset_target_weight, + # avg_factor=avg_factor * 2, + # ) loss_offset = self.loss_offset( - offset_pred, offset_target, wh_offset_target_weight, avg_factor=avg_factor * 2 + offset_pred, + offset_target, + wh_offset_target_weight, + avg_factor=avg_factor * 2, ) # loss_pairing = self.loss_pairing( # center2vertex_pred, @@ -208,15 +243,21 @@ def loss( # avg_factor=avg_factor, # ) loss_c2v = self.loss_c2v( - center2vertex_pred, center2vertex_target, c2v_v2c_target_weight, avg_factor=avg_factor * 8 + center2vertex_pred, + center2vertex_target, + c2v_v2c_target_weight, + avg_factor=avg_factor * 8, ) loss_v2c = self.loss_v2c( - vertex2center_pred, vertex2center_target, c2v_v2c_target_weight, avg_factor=avg_factor * 8 + vertex2center_pred, + vertex2center_target, + c2v_v2c_target_weight, + avg_factor=avg_factor * 8, ) return dict( loss_center_heatmap=loss_center_heatmap, - loss_wh=loss_wh, + # loss_wh=loss_wh, loss_offset=loss_offset, # loss_pairing=loss_pairing, loss_c2v=loss_c2v, @@ -255,12 +296,16 @@ def get_targets(self, gt_bboxes, gt_labels, feat_shape, img_shape): width_ratio = float(feat_w / img_w) height_ratio = float(feat_h / img_h) - center_heatmap_target = gt_bboxes[-1].new_zeros([bs, self.num_classes, feat_h, feat_w]) - wh_target = gt_bboxes[-1].new_zeros([bs, 2, feat_h, feat_w]) + center_heatmap_target = gt_bboxes[-1].new_zeros( + [bs, 2 * self.num_classes, feat_h, feat_w] + ) + # wh_target = gt_bboxes[-1].new_zeros([bs, 2, feat_h, feat_w]) offset_target = gt_bboxes[-1].new_zeros([bs, 2, feat_h, feat_w]) - wh_offset_target_weight = gt_bboxes[-1].new_zeros([bs, 2, feat_h, feat_w]) - center2vertex_target = gt_bboxes[-1].new_zeros([bs, 8, feat_h, feat_w]) - vertex2center_target = gt_bboxes[-1].new_zeros([bs, 8, feat_h, feat_w]) + wh_offset_target_weight = gt_bboxes[-1].new_zeros( + [bs, 2, feat_h, feat_w] + ) + c2v_target = gt_bboxes[-1].new_zeros([bs, 8, feat_h, feat_w]) + v2c_target = gt_bboxes[-1].new_zeros([bs, 8, feat_h, feat_w]) c2v_v2c_target_weight = gt_bboxes[-1].new_zeros([bs, 8, feat_h, feat_w]) for batch_id in range(bs): @@ -275,57 +320,128 @@ def get_targets(self, gt_bboxes, gt_labels, feat_shape, img_shape): ctx, cty = ct scale_box_h = (gt_bbox[j][3] - gt_bbox[j][1]) * height_ratio scale_box_w = (gt_bbox[j][2] - gt_bbox[j][0]) * width_ratio - radius = gaussian_radius([scale_box_h, scale_box_w], min_overlap=0.3) + radius = gaussian_radius( + [scale_box_h, scale_box_w], min_overlap=0.3 + ) radius = max(0, int(radius)) ind = gt_label[j] - gen_gaussian_target(center_heatmap_target[batch_id, ind], [ctx_int, cty_int], radius) + gen_gaussian_target( + center_heatmap_target[batch_id, ind], + [ctx_int, cty_int], + radius, + ) - wh_target[batch_id, 0, cty_int, ctx_int] = scale_box_w - wh_target[batch_id, 1, cty_int, ctx_int] = scale_box_h + tl_x, tl_y = ( + gt_bbox[j][0] * width_ratio, + gt_bbox[j][1] * height_ratio, + ) + tr_x, tr_y = ( + gt_bbox[j][2] * width_ratio, + gt_bbox[j][1] * height_ratio, + ) + br_x, br_y = ( + gt_bbox[j][2] * width_ratio, + gt_bbox[j][3] * height_ratio, + ) + bl_x, bl_y = ( + gt_bbox[j][0] * width_ratio, + gt_bbox[j][3] * height_ratio, + ) + + tl_x_int, tl_y_int = int(tl_x), int(tl_y) + tr_x_int, tr_y_int = int(tr_x), int(tr_y) + br_x_int, br_y_int = int(br_x), int(br_y) + bl_x_int, bl_y_int = int(bl_x), int(bl_y) + + tl_x_int = tl_x_int if 0 <= tl_x_int else 0 + tr_x_int = tr_x_int if 0 <= tr_x_int else 0 + br_x_int = br_x_int if 0 <= br_x_int else 0 + bl_x_int = bl_x_int if 0 <= bl_x_int else 0 + tl_x_int = tl_x_int if tl_x_int < feat_w else feat_w - 1 + tr_x_int = tr_x_int if tr_x_int < feat_w else feat_w - 1 + br_x_int = br_x_int if br_x_int < feat_w else feat_w - 1 + bl_x_int = bl_x_int if bl_x_int < feat_w else feat_w - 1 + tl_y_int = tl_y_int if 0 <= tl_y_int else 0 + tr_y_int = tr_y_int if 0 <= tr_y_int else 0 + br_y_int = br_y_int if 0 <= br_y_int else 0 + bl_y_int = bl_y_int if 0 <= bl_y_int else 0 + tl_y_int = tl_y_int if tl_y_int < feat_h else feat_h - 1 + tr_y_int = tr_y_int if tr_y_int < feat_h else feat_h - 1 + br_y_int = br_y_int if br_y_int < feat_h else feat_h - 1 + bl_y_int = bl_y_int if bl_y_int < feat_h else feat_h - 1 + for x, y in ( + (tl_x_int, tl_y_int), + (tr_x_int, tr_y_int), + (br_x_int, br_y_int), + (bl_x_int, bl_y_int), + ): + gen_gaussian_target( + center_heatmap_target[batch_id, 2 * ind], + [x, y], + radius, + ) + + # wh_target[batch_id, 0, cty_int, ctx_int] = scale_box_w + # wh_target[batch_id, 1, cty_int, ctx_int] = scale_box_h offset_target[batch_id, 0, cty_int, ctx_int] = ctx - ctx_int + offset_target[batch_id, 0, tl_x_int, tl_y_int] = tl_x - tl_x_int + offset_target[batch_id, 0, tr_x_int, tr_y_int] = tr_x - tr_x_int + offset_target[batch_id, 0, br_x_int, br_y_int] = br_x - br_x_int + offset_target[batch_id, 0, bl_x_int, bl_y_int] = bl_x - bl_x_int + offset_target[batch_id, 1, cty_int, ctx_int] = cty - cty_int + offset_target[batch_id, 1, tl_x_int, tl_y_int] = tl_y - tl_y_int + offset_target[batch_id, 1, tr_x_int, tr_y_int] = tr_y - tr_y_int + offset_target[batch_id, 1, br_x_int, br_y_int] = br_y - br_y_int + offset_target[batch_id, 1, bl_x_int, bl_y_int] = bl_y - bl_y_int wh_offset_target_weight[batch_id, :, cty_int, ctx_int] = 1 - center2vertex_target[batch_id, 0, cty_int, ctx_int] = -scale_box_w / 2 - center2vertex_target[batch_id, 1, cty_int, ctx_int] = -scale_box_h / 2 - center2vertex_target[batch_id, 2, cty_int, ctx_int] = scale_box_w / 2 - center2vertex_target[batch_id, 3, cty_int, ctx_int] = -scale_box_h / 2 - center2vertex_target[batch_id, 4, cty_int, ctx_int] = scale_box_w / 2 - center2vertex_target[batch_id, 5, cty_int, ctx_int] = scale_box_h / 2 - center2vertex_target[batch_id, 6, cty_int, ctx_int] = -scale_box_w / 2 - center2vertex_target[batch_id, 7, cty_int, ctx_int] = scale_box_h / 2 - - vertex2center_target[batch_id, 0, cty_int, ctx_int] = scale_box_w / 2 - vertex2center_target[batch_id, 1, cty_int, ctx_int] = scale_box_h / 2 - vertex2center_target[batch_id, 2, cty_int, ctx_int] = -scale_box_w / 2 - vertex2center_target[batch_id, 3, cty_int, ctx_int] = scale_box_h / 2 - vertex2center_target[batch_id, 4, cty_int, ctx_int] = -scale_box_w / 2 - vertex2center_target[batch_id, 5, cty_int, ctx_int] = -scale_box_h / 2 - vertex2center_target[batch_id, 6, cty_int, ctx_int] = scale_box_w / 2 - vertex2center_target[batch_id, 7, cty_int, ctx_int] = -scale_box_h / 2 + c2v_target[batch_id, 0, cty_int, ctx_int] = -scale_box_w / 2 + c2v_target[batch_id, 1, cty_int, ctx_int] = -scale_box_h / 2 + c2v_target[batch_id, 2, cty_int, ctx_int] = scale_box_w / 2 + c2v_target[batch_id, 3, cty_int, ctx_int] = -scale_box_h / 2 + c2v_target[batch_id, 4, cty_int, ctx_int] = scale_box_w / 2 + c2v_target[batch_id, 5, cty_int, ctx_int] = scale_box_h / 2 + c2v_target[batch_id, 6, cty_int, ctx_int] = -scale_box_w / 2 + c2v_target[batch_id, 7, cty_int, ctx_int] = scale_box_h / 2 + + v2c_target[batch_id, 0, tl_x_int, tl_y_int] = scale_box_w / 2 + v2c_target[batch_id, 1, tl_x_int, tl_y_int] = scale_box_h / 2 + v2c_target[batch_id, 2, tr_x_int, tr_y_int] = -scale_box_w / 2 + v2c_target[batch_id, 3, tr_x_int, tr_y_int] = scale_box_h / 2 + v2c_target[batch_id, 4, br_x_int, br_y_int] = -scale_box_w / 2 + v2c_target[batch_id, 5, br_x_int, br_y_int] = -scale_box_h / 2 + v2c_target[batch_id, 6, bl_x_int, bl_y_int] = scale_box_w / 2 + v2c_target[batch_id, 7, bl_x_int, bl_y_int] = -scale_box_h / 2 c2v_v2c_target_weight[batch_id, :, cty_int, ctx_int] = 1 avg_factor = max(1, center_heatmap_target.eq(1).sum()) target_result = dict( center_heatmap_target=center_heatmap_target, - wh_target=wh_target, + # wh_target=wh_target, offset_target=offset_target, wh_offset_target_weight=wh_offset_target_weight, - center2vertex_target=center2vertex_target, - vertex2center_target=vertex2center_target, + center2vertex_target=c2v_target, + vertex2center_target=v2c_target, c2v_v2c_target_weight=c2v_v2c_target_weight, ) return target_result, avg_factor @force_fp32( - apply_to=("center_heatmap_preds", "wh_preds", "offset_preds", "center2vertex_preds", "vertex2center_preds") + apply_to=( + "center_heatmap_preds", + # "wh_preds", + "offset_preds", + "center2vertex_preds", + "vertex2center_preds", + ) ) def get_bboxes( self, center_heatmap_preds, - wh_preds, + # wh_preds, offset_preds, center2vertex_preds, vertex2center_preds, @@ -363,20 +479,45 @@ def get_bboxes( """ assert ( len(center_heatmap_preds) - == len(wh_preds) + # == len(wh_preds) == len(offset_preds) == len(center2vertex_preds) == len(vertex2center_preds) == 1 ) - diff = (center2vertex_preds[0] - vertex2center_preds[0]) / 2 - wh_preds[0][:, 0, ...] = (-diff[:, 0, ...] + diff[:, 2, ...] + diff[:, 4, ...] - diff[:, 6, ...]) / 2 - wh_preds[0][:, 1, ...] = (-diff[:, 1, ...] - diff[:, 3, ...] + diff[:, 5, ...] - diff[:, 7, ...]) / 2 + wh_preds = [torch.zeros_like(offset_preds[0])] + # diff = (center2vertex_preds[0] - vertex2center_preds[0]) / 2 + # wh_preds[0][:, 0, ...] = ( + # -diff[:, 0, ...] + # + diff[:, 2, ...] + # + diff[:, 4, ...] + # - diff[:, 6, ...] + # ) / 2 + # wh_preds[0][:, 1, ...] = ( + # -diff[:, 1, ...] + # - diff[:, 3, ...] + # + diff[:, 5, ...] + # - diff[:, 7, ...] + # ) / 2 + wh_preds[0][:, 0, ...] = ( + -center2vertex_preds[:, 0, ...] + + center2vertex_preds[:, 2, ...] + + center2vertex_preds[:, 4, ...] + - center2vertex_preds[:, 6, ...] + ) / 2 + wh_preds[0][:, 1, ...] = ( + -center2vertex_preds[:, 1, ...] + - center2vertex_preds[:, 3, ...] + + center2vertex_preds[:, 5, ...] + - center2vertex_preds[:, 7, ...] + ) / 2 result_list = [] for img_id in range(len(img_metas)): result_list.append( self._get_bboxes_single( - center_heatmap_preds[0][img_id : img_id + 1, ...], + center_heatmap_preds[0][ + img_id : img_id + 1, 0, ... + ], ### 000!!!! wh_preds[0][img_id : img_id + 1, ...], offset_preds[0][img_id : img_id + 1, ...], img_metas[img_id], @@ -434,17 +575,31 @@ def _get_bboxes_single( det_bboxes = batch_det_bboxes.view([-1, 5]) det_labels = batch_labels.view(-1) - batch_border = det_bboxes.new_tensor(img_meta["border"])[..., [2, 0, 2, 0]] + batch_border = det_bboxes.new_tensor(img_meta["border"])[ + ..., [2, 0, 2, 0] + ] det_bboxes[..., :4] -= batch_border if rescale: - det_bboxes[..., :4] /= det_bboxes.new_tensor(img_meta["scale_factor"]) + det_bboxes[..., :4] /= det_bboxes.new_tensor( + img_meta["scale_factor"] + ) if with_nms: - det_bboxes, det_labels = self._bboxes_nms(det_bboxes, det_labels, self.test_cfg) + det_bboxes, det_labels = self._bboxes_nms( + det_bboxes, det_labels, self.test_cfg + ) return det_bboxes, det_labels - def decode_heatmap(self, center_heatmap_pred, wh_pred, offset_pred, img_shape, k=100, kernel=3): + def decode_heatmap( + self, + center_heatmap_pred, + wh_pred, + offset_pred, + img_shape, + k=100, + kernel=3, + ): """Transform outputs into detections raw bbox prediction. Args: @@ -468,9 +623,13 @@ def decode_heatmap(self, center_heatmap_pred, wh_pred, offset_pred, img_shape, k height, width = center_heatmap_pred.shape[2:] inp_h, inp_w = img_shape - center_heatmap_pred = get_local_maximum(center_heatmap_pred, kernel=kernel) + center_heatmap_pred = get_local_maximum( + center_heatmap_pred, kernel=kernel + ) - *batch_dets, topk_ys, topk_xs = get_topk_from_heatmap(center_heatmap_pred, k=k) + *batch_dets, topk_ys, topk_xs = get_topk_from_heatmap( + center_heatmap_pred, k=k + ) batch_scores, batch_index, batch_topk_labels = batch_dets wh = transpose_and_gather_feat(wh_pred, batch_index) @@ -483,13 +642,17 @@ def decode_heatmap(self, center_heatmap_pred, wh_pred, offset_pred, img_shape, k br_y = (topk_ys + wh[..., 1] / 2) * (inp_h / height) batch_bboxes = torch.stack([tl_x, tl_y, br_x, br_y], dim=2) - batch_bboxes = torch.cat((batch_bboxes, batch_scores[..., None]), dim=-1) + batch_bboxes = torch.cat( + (batch_bboxes, batch_scores[..., None]), dim=-1 + ) return batch_bboxes, batch_topk_labels def _bboxes_nms(self, bboxes, labels, cfg): if labels.numel() > 0: max_num = cfg.max_per_img - bboxes, keep = batched_nms(bboxes[:, :4], bboxes[:, -1].contiguous(), labels, cfg.nms) + bboxes, keep = batched_nms( + bboxes[:, :4], bboxes[:, -1].contiguous(), labels, cfg.nms + ) if max_num > 0: bboxes = bboxes[:max_num] labels = labels[keep][:max_num] From 33916663f341805000de1006f255e1380799333f Mon Sep 17 00:00:00 2001 From: ArchieAlexArkhipov Date: Mon, 8 May 2023 13:19:28 +0000 Subject: [PATCH 05/25] correct 2 heatmap l1-loss --- .../dense_heads/cycle_centernet_head.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/mmdet/models/dense_heads/cycle_centernet_head.py b/mmdet/models/dense_heads/cycle_centernet_head.py index 14b7e083..410b3194 100644 --- a/mmdet/models/dense_heads/cycle_centernet_head.py +++ b/mmdet/models/dense_heads/cycle_centernet_head.py @@ -246,13 +246,13 @@ def loss( center2vertex_pred, center2vertex_target, c2v_v2c_target_weight, - avg_factor=avg_factor * 8, + avg_factor=avg_factor * 8, # 8, ) loss_v2c = self.loss_v2c( vertex2center_pred, vertex2center_target, c2v_v2c_target_weight, - avg_factor=avg_factor * 8, + avg_factor=avg_factor * 8, # 8, ) return dict( @@ -500,23 +500,23 @@ def get_bboxes( # - diff[:, 7, ...] # ) / 2 wh_preds[0][:, 0, ...] = ( - -center2vertex_preds[:, 0, ...] - + center2vertex_preds[:, 2, ...] - + center2vertex_preds[:, 4, ...] - - center2vertex_preds[:, 6, ...] + -center2vertex_preds[0][:, 0, ...] + + center2vertex_preds[0][:, 2, ...] + + center2vertex_preds[0][:, 4, ...] + - center2vertex_preds[0][:, 6, ...] ) / 2 wh_preds[0][:, 1, ...] = ( - -center2vertex_preds[:, 1, ...] - - center2vertex_preds[:, 3, ...] - + center2vertex_preds[:, 5, ...] - - center2vertex_preds[:, 7, ...] + -center2vertex_preds[0][:, 1, ...] + - center2vertex_preds[0][:, 3, ...] + + center2vertex_preds[0][:, 5, ...] + - center2vertex_preds[0][:, 7, ...] ) / 2 result_list = [] for img_id in range(len(img_metas)): result_list.append( self._get_bboxes_single( center_heatmap_preds[0][ - img_id : img_id + 1, 0, ... + img_id : img_id + 1, 0:1, ... ], ### 000!!!! wh_preds[0][img_id : img_id + 1, ...], offset_preds[0][img_id : img_id + 1, ...], From 2eb9abf536e7e91a8e4f22ec0efddbec499c0b51 Mon Sep 17 00:00:00 2001 From: ArchieAlexArkhipov Date: Mon, 8 May 2023 17:28:49 +0000 Subject: [PATCH 06/25] fix x vs y in get_targets --- .../dense_heads/cycle_centernet_head.py | 37 +++++++++++-------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/mmdet/models/dense_heads/cycle_centernet_head.py b/mmdet/models/dense_heads/cycle_centernet_head.py index 410b3194..5cade61b 100644 --- a/mmdet/models/dense_heads/cycle_centernet_head.py +++ b/mmdet/models/dense_heads/cycle_centernet_head.py @@ -385,16 +385,16 @@ def get_targets(self, gt_bboxes, gt_labels, feat_shape, img_shape): # wh_target[batch_id, 1, cty_int, ctx_int] = scale_box_h offset_target[batch_id, 0, cty_int, ctx_int] = ctx - ctx_int - offset_target[batch_id, 0, tl_x_int, tl_y_int] = tl_x - tl_x_int - offset_target[batch_id, 0, tr_x_int, tr_y_int] = tr_x - tr_x_int - offset_target[batch_id, 0, br_x_int, br_y_int] = br_x - br_x_int - offset_target[batch_id, 0, bl_x_int, bl_y_int] = bl_x - bl_x_int + offset_target[batch_id, 0, tl_y_int, tl_x_int] = tl_x - tl_x_int + offset_target[batch_id, 0, tr_y_int, tr_x_int] = tr_x - tr_x_int + offset_target[batch_id, 0, br_y_int, br_x_int] = br_x - br_x_int + offset_target[batch_id, 0, bl_y_int, bl_x_int] = bl_x - bl_x_int offset_target[batch_id, 1, cty_int, ctx_int] = cty - cty_int - offset_target[batch_id, 1, tl_x_int, tl_y_int] = tl_y - tl_y_int - offset_target[batch_id, 1, tr_x_int, tr_y_int] = tr_y - tr_y_int - offset_target[batch_id, 1, br_x_int, br_y_int] = br_y - br_y_int - offset_target[batch_id, 1, bl_x_int, bl_y_int] = bl_y - bl_y_int + offset_target[batch_id, 1, tl_y_int, tl_x_int] = tl_y - tl_y_int + offset_target[batch_id, 1, tr_y_int, tr_x_int] = tr_y - tr_y_int + offset_target[batch_id, 1, br_y_int, br_x_int] = br_y - br_y_int + offset_target[batch_id, 1, bl_y_int, bl_x_int] = bl_y - bl_y_int wh_offset_target_weight[batch_id, :, cty_int, ctx_int] = 1 @@ -407,16 +407,21 @@ def get_targets(self, gt_bboxes, gt_labels, feat_shape, img_shape): c2v_target[batch_id, 6, cty_int, ctx_int] = -scale_box_w / 2 c2v_target[batch_id, 7, cty_int, ctx_int] = scale_box_h / 2 - v2c_target[batch_id, 0, tl_x_int, tl_y_int] = scale_box_w / 2 - v2c_target[batch_id, 1, tl_x_int, tl_y_int] = scale_box_h / 2 - v2c_target[batch_id, 2, tr_x_int, tr_y_int] = -scale_box_w / 2 - v2c_target[batch_id, 3, tr_x_int, tr_y_int] = scale_box_h / 2 - v2c_target[batch_id, 4, br_x_int, br_y_int] = -scale_box_w / 2 - v2c_target[batch_id, 5, br_x_int, br_y_int] = -scale_box_h / 2 - v2c_target[batch_id, 6, bl_x_int, bl_y_int] = scale_box_w / 2 - v2c_target[batch_id, 7, bl_x_int, bl_y_int] = -scale_box_h / 2 + v2c_target[batch_id, 0, tl_y_int, tl_x_int] = scale_box_w / 2 + v2c_target[batch_id, 1, tl_y_int, tl_x_int] = scale_box_h / 2 + v2c_target[batch_id, 2, tr_y_int, tr_x_int] = -scale_box_w / 2 + v2c_target[batch_id, 3, tr_y_int, tr_x_int] = scale_box_h / 2 + v2c_target[batch_id, 4, br_y_int, br_x_int] = -scale_box_w / 2 + v2c_target[batch_id, 5, br_y_int, br_x_int] = -scale_box_h / 2 + v2c_target[batch_id, 6, bl_y_int, bl_x_int] = scale_box_w / 2 + v2c_target[batch_id, 7, bl_y_int, bl_x_int] = -scale_box_h / 2 c2v_v2c_target_weight[batch_id, :, cty_int, ctx_int] = 1 + c2v_v2c_target_weight[batch_id, :, tl_y_int, tl_x_int] = 1 + c2v_v2c_target_weight[batch_id, :, tr_y_int, tr_x_int] = 1 + c2v_v2c_target_weight[batch_id, :, br_y_int, br_x_int] = 1 + c2v_v2c_target_weight[batch_id, :, bl_y_int, bl_x_int] = 1 + avg_factor = max(1, center_heatmap_target.eq(1).sum()) target_result = dict( center_heatmap_target=center_heatmap_target, From 58ee39f47ea7021d88e804b63b75e0159a16362c Mon Sep 17 00:00:00 2001 From: ArchieAlexArkhipov Date: Mon, 8 May 2023 23:34:51 +0000 Subject: [PATCH 07/25] beta pairing loss --- .../dense_heads/cycle_centernet_head.py | 150 +++++++++++++++++- 1 file changed, 148 insertions(+), 2 deletions(-) diff --git a/mmdet/models/dense_heads/cycle_centernet_head.py b/mmdet/models/dense_heads/cycle_centernet_head.py index 5cade61b..40b113ae 100644 --- a/mmdet/models/dense_heads/cycle_centernet_head.py +++ b/mmdet/models/dense_heads/cycle_centernet_head.py @@ -1,6 +1,7 @@ # Copyright (c) OpenMMLab. All rights reserved. import torch import torch.nn as nn +from numpy import pi, exp from mmcv.cnn import bias_init_with_prob, normal_init from mmcv.ops import batched_nms from mmcv.runner import force_fp32 @@ -242,17 +243,27 @@ def loss( # ..., # avg_factor=avg_factor, # ) + c2v_v2c_target_weight = self._dynamic_weighing( + gt_bboxes, + center_heatmap_pred.shape, + img_metas[0]["pad_shape"], + center2vertex_pred, + vertex2center_pred, + center2vertex_target, + center2vertex_target, + c2v_v2c_target_weight, + ) loss_c2v = self.loss_c2v( center2vertex_pred, center2vertex_target, c2v_v2c_target_weight, - avg_factor=avg_factor * 8, # 8, + avg_factor=avg_factor * 8, ) loss_v2c = self.loss_v2c( vertex2center_pred, vertex2center_target, c2v_v2c_target_weight, - avg_factor=avg_factor * 8, # 8, + avg_factor=avg_factor * 8, ) return dict( @@ -264,6 +275,141 @@ def loss( loss_v2c=loss_v2c, ) + def _dynamic_weighing( + self, + gt_bboxes, + feat_shape, + img_shape, + c2v_pred, + v2c_pred, + c2v_target, + v2c_target, + c2v_v2c_target_weight, + ): + img_h, img_w = img_shape[:2] + bs, _, feat_h, feat_w = feat_shape + + width_ratio = float(feat_w / img_w) + height_ratio = float(feat_h / img_h) + + for batch_id in range(bs): + gt_bbox = gt_bboxes[batch_id] + center_x = (gt_bbox[:, [0]] + gt_bbox[:, [2]]) * width_ratio / 2 + center_y = (gt_bbox[:, [1]] + gt_bbox[:, [3]]) * height_ratio / 2 + gt_centers = torch.cat((center_x, center_y), dim=1) + + for j, ct in enumerate(gt_centers): + ctx_int, cty_int = ct.int() + tl_x, tl_y = ( + gt_bbox[j][0] * width_ratio, + gt_bbox[j][1] * height_ratio, + ) + tr_x, tr_y = ( + gt_bbox[j][2] * width_ratio, + gt_bbox[j][1] * height_ratio, + ) + br_x, br_y = ( + gt_bbox[j][2] * width_ratio, + gt_bbox[j][3] * height_ratio, + ) + bl_x, bl_y = ( + gt_bbox[j][0] * width_ratio, + gt_bbox[j][3] * height_ratio, + ) + + tl_x_int, tl_y_int = int(tl_x), int(tl_y) + tr_x_int, tr_y_int = int(tr_x), int(tr_y) + br_x_int, br_y_int = int(br_x), int(br_y) + bl_x_int, bl_y_int = int(bl_x), int(bl_y) + + tl_x_int = tl_x_int if 0 <= tl_x_int else 0 + tr_x_int = tr_x_int if 0 <= tr_x_int else 0 + br_x_int = br_x_int if 0 <= br_x_int else 0 + bl_x_int = bl_x_int if 0 <= bl_x_int else 0 + tl_x_int = tl_x_int if tl_x_int < feat_w else feat_w - 1 + tr_x_int = tr_x_int if tr_x_int < feat_w else feat_w - 1 + br_x_int = br_x_int if br_x_int < feat_w else feat_w - 1 + bl_x_int = bl_x_int if bl_x_int < feat_w else feat_w - 1 + tl_y_int = tl_y_int if 0 <= tl_y_int else 0 + tr_y_int = tr_y_int if 0 <= tr_y_int else 0 + br_y_int = br_y_int if 0 <= br_y_int else 0 + bl_y_int = bl_y_int if 0 <= bl_y_int else 0 + tl_y_int = tl_y_int if tl_y_int < feat_h else feat_h - 1 + tr_y_int = tr_y_int if tr_y_int < feat_h else feat_h - 1 + br_y_int = br_y_int if br_y_int < feat_h else feat_h - 1 + bl_y_int = bl_y_int if bl_y_int < feat_h else feat_h - 1 + + ctx, cty = ct + for idx, v_x_y in enumerate( + ( + (tl_x_int, tl_y_int), + (tr_x_int, tr_y_int), + (br_x_int, br_y_int), + (bl_x_int, bl_y_int), + ) + ): + v_x_int, v_y_int = v_x_y + D_cv = min( + torch.tensor(1.0), + ( + torch.abs( + c2v_pred[batch_id, 2 * idx, cty_int, ctx_int] + - c2v_target[ + batch_id, 2 * idx, cty_int, ctx_int + ] + ) + + torch.abs( + v2c_pred[batch_id, 2 * idx, v_y_int, v_x_int] + - v2c_target[ + batch_id, 2 * idx, v_y_int, v_x_int + ] + ) + ) + / torch.abs( + c2v_target[batch_id, 2 * idx, cty_int, ctx_int] + ), + ) + w = 1 - torch.exp(-pi * D_cv) + c2v_v2c_target_weight[ + batch_id, 2 * idx, cty_int, ctx_int + ] += w + c2v_v2c_target_weight[ + batch_id, 2 * idx, v_y_int, v_x_int + ] += w + D_cv = min( + torch.tensor(1.0), + ( + torch.abs( + c2v_pred[ + batch_id, 2 * idx + 1, cty_int, ctx_int + ] + - c2v_target[ + batch_id, 2 * idx + 1, cty_int, ctx_int + ] + ) + + torch.abs( + v2c_pred[ + batch_id, 2 * idx + 1, v_y_int, v_x_int + ] + - v2c_target[ + batch_id, 2 * idx + 1, v_y_int, v_x_int + ] + ) + ) + / torch.abs( + c2v_target[batch_id, 2 * idx + 1, cty_int, ctx_int] + ), + ) + w = 1 - torch.exp(-pi * D_cv) + c2v_v2c_target_weight[ + batch_id, 2 * idx + 1, cty_int, ctx_int + ] += w + c2v_v2c_target_weight[ + batch_id, 2 * idx + 1, v_y_int, v_x_int + ] += w + + return c2v_v2c_target_weight + def get_targets(self, gt_bboxes, gt_labels, feat_shape, img_shape): """Compute regression and classification targets in multiple images. From d5f429566056aedf2b4884833ce6fb6d948fc4ee Mon Sep 17 00:00:00 2001 From: ArchieAlexArkhipov Date: Thu, 11 May 2023 13:05:26 +0000 Subject: [PATCH 08/25] Pairing loss wrong --- .../dense_heads/cycle_centernet_head.py | 278 ++++-------------- 1 file changed, 59 insertions(+), 219 deletions(-) diff --git a/mmdet/models/dense_heads/cycle_centernet_head.py b/mmdet/models/dense_heads/cycle_centernet_head.py index 40b113ae..50771317 100644 --- a/mmdet/models/dense_heads/cycle_centernet_head.py +++ b/mmdet/models/dense_heads/cycle_centernet_head.py @@ -57,9 +57,7 @@ def __init__( ): super(CycleCenterNetHeadL1, self).__init__(init_cfg) self.num_classes = num_classes - self.heatmap_head = self._build_head( - in_channel, feat_channel, 2 * num_classes - ) + self.heatmap_head = self._build_head(in_channel, feat_channel, 2 * num_classes) # self.wh_head = self._build_head(in_channel, feat_channel, 2) self.offset_head = self._build_head(in_channel, feat_channel, 2) self.center2vertex_head = self._build_head(in_channel, feat_channel, 8) @@ -144,7 +142,6 @@ def forward_single(self, feat): @force_fp32( apply_to=( "center_heatmap_preds", - # "wh_preds", "offset_preds", "center2vertex_pred", "vertex2center_pred", @@ -153,7 +150,6 @@ def forward_single(self, feat): def loss( self, center_heatmap_preds, - # wh_preds, offset_preds, center2vertex_pred, vertex2center_pred, @@ -198,7 +194,6 @@ def loss( == 1 ) center_heatmap_pred = center_heatmap_preds[0] - # wh_pred = wh_preds[0] offset_pred = offset_preds[0] center2vertex_pred = center2vertex_pred[0] vertex2center_pred = vertex2center_pred[0] @@ -208,209 +203,50 @@ def loss( gt_labels, center_heatmap_pred.shape, img_metas[0]["pad_shape"], + center2vertex_pred, + vertex2center_pred, ) center_heatmap_target = target_result["center_heatmap_target"] # wh_target = target_result["wh_target"] offset_target = target_result["offset_target"] - wh_offset_target_weight = target_result["wh_offset_target_weight"] + offset_target_weight = target_result["offset_target_weight"] center2vertex_target = target_result["center2vertex_target"] vertex2center_target = target_result["vertex2center_target"] - c2v_v2c_target_weight = target_result["c2v_v2c_target_weight"] + pairing_weight = target_result["pairing_weight"] # Since the channel of wh_target and offset_target is 2, the avg_factor # of loss_center_heatmap is always 1/2 of loss_wh and loss_offset. loss_center_heatmap = self.loss_center_heatmap( center_heatmap_pred, center_heatmap_target, avg_factor=avg_factor ) - # loss_wh = self.loss_wh( - # wh_pred, - # wh_target, - # wh_offset_target_weight, - # avg_factor=avg_factor * 2, - # ) loss_offset = self.loss_offset( offset_pred, offset_target, - wh_offset_target_weight, + offset_target_weight, avg_factor=avg_factor * 2, ) - # loss_pairing = self.loss_pairing( - # center2vertex_pred, - # vertex2center_pred, - # center2vertex_target, - # vertex2center_target, - # ..., - # avg_factor=avg_factor, - # ) - c2v_v2c_target_weight = self._dynamic_weighing( - gt_bboxes, - center_heatmap_pred.shape, - img_metas[0]["pad_shape"], - center2vertex_pred, - vertex2center_pred, - center2vertex_target, - center2vertex_target, - c2v_v2c_target_weight, - ) loss_c2v = self.loss_c2v( center2vertex_pred, center2vertex_target, - c2v_v2c_target_weight, + pairing_weight, avg_factor=avg_factor * 8, ) loss_v2c = self.loss_v2c( vertex2center_pred, vertex2center_target, - c2v_v2c_target_weight, + pairing_weight, avg_factor=avg_factor * 8, ) return dict( loss_center_heatmap=loss_center_heatmap, - # loss_wh=loss_wh, loss_offset=loss_offset, - # loss_pairing=loss_pairing, loss_c2v=loss_c2v, loss_v2c=loss_v2c, ) - def _dynamic_weighing( - self, - gt_bboxes, - feat_shape, - img_shape, - c2v_pred, - v2c_pred, - c2v_target, - v2c_target, - c2v_v2c_target_weight, - ): - img_h, img_w = img_shape[:2] - bs, _, feat_h, feat_w = feat_shape - - width_ratio = float(feat_w / img_w) - height_ratio = float(feat_h / img_h) - - for batch_id in range(bs): - gt_bbox = gt_bboxes[batch_id] - center_x = (gt_bbox[:, [0]] + gt_bbox[:, [2]]) * width_ratio / 2 - center_y = (gt_bbox[:, [1]] + gt_bbox[:, [3]]) * height_ratio / 2 - gt_centers = torch.cat((center_x, center_y), dim=1) - - for j, ct in enumerate(gt_centers): - ctx_int, cty_int = ct.int() - tl_x, tl_y = ( - gt_bbox[j][0] * width_ratio, - gt_bbox[j][1] * height_ratio, - ) - tr_x, tr_y = ( - gt_bbox[j][2] * width_ratio, - gt_bbox[j][1] * height_ratio, - ) - br_x, br_y = ( - gt_bbox[j][2] * width_ratio, - gt_bbox[j][3] * height_ratio, - ) - bl_x, bl_y = ( - gt_bbox[j][0] * width_ratio, - gt_bbox[j][3] * height_ratio, - ) - - tl_x_int, tl_y_int = int(tl_x), int(tl_y) - tr_x_int, tr_y_int = int(tr_x), int(tr_y) - br_x_int, br_y_int = int(br_x), int(br_y) - bl_x_int, bl_y_int = int(bl_x), int(bl_y) - - tl_x_int = tl_x_int if 0 <= tl_x_int else 0 - tr_x_int = tr_x_int if 0 <= tr_x_int else 0 - br_x_int = br_x_int if 0 <= br_x_int else 0 - bl_x_int = bl_x_int if 0 <= bl_x_int else 0 - tl_x_int = tl_x_int if tl_x_int < feat_w else feat_w - 1 - tr_x_int = tr_x_int if tr_x_int < feat_w else feat_w - 1 - br_x_int = br_x_int if br_x_int < feat_w else feat_w - 1 - bl_x_int = bl_x_int if bl_x_int < feat_w else feat_w - 1 - tl_y_int = tl_y_int if 0 <= tl_y_int else 0 - tr_y_int = tr_y_int if 0 <= tr_y_int else 0 - br_y_int = br_y_int if 0 <= br_y_int else 0 - bl_y_int = bl_y_int if 0 <= bl_y_int else 0 - tl_y_int = tl_y_int if tl_y_int < feat_h else feat_h - 1 - tr_y_int = tr_y_int if tr_y_int < feat_h else feat_h - 1 - br_y_int = br_y_int if br_y_int < feat_h else feat_h - 1 - bl_y_int = bl_y_int if bl_y_int < feat_h else feat_h - 1 - - ctx, cty = ct - for idx, v_x_y in enumerate( - ( - (tl_x_int, tl_y_int), - (tr_x_int, tr_y_int), - (br_x_int, br_y_int), - (bl_x_int, bl_y_int), - ) - ): - v_x_int, v_y_int = v_x_y - D_cv = min( - torch.tensor(1.0), - ( - torch.abs( - c2v_pred[batch_id, 2 * idx, cty_int, ctx_int] - - c2v_target[ - batch_id, 2 * idx, cty_int, ctx_int - ] - ) - + torch.abs( - v2c_pred[batch_id, 2 * idx, v_y_int, v_x_int] - - v2c_target[ - batch_id, 2 * idx, v_y_int, v_x_int - ] - ) - ) - / torch.abs( - c2v_target[batch_id, 2 * idx, cty_int, ctx_int] - ), - ) - w = 1 - torch.exp(-pi * D_cv) - c2v_v2c_target_weight[ - batch_id, 2 * idx, cty_int, ctx_int - ] += w - c2v_v2c_target_weight[ - batch_id, 2 * idx, v_y_int, v_x_int - ] += w - D_cv = min( - torch.tensor(1.0), - ( - torch.abs( - c2v_pred[ - batch_id, 2 * idx + 1, cty_int, ctx_int - ] - - c2v_target[ - batch_id, 2 * idx + 1, cty_int, ctx_int - ] - ) - + torch.abs( - v2c_pred[ - batch_id, 2 * idx + 1, v_y_int, v_x_int - ] - - v2c_target[ - batch_id, 2 * idx + 1, v_y_int, v_x_int - ] - ) - ) - / torch.abs( - c2v_target[batch_id, 2 * idx + 1, cty_int, ctx_int] - ), - ) - w = 1 - torch.exp(-pi * D_cv) - c2v_v2c_target_weight[ - batch_id, 2 * idx + 1, cty_int, ctx_int - ] += w - c2v_v2c_target_weight[ - batch_id, 2 * idx + 1, v_y_int, v_x_int - ] += w - - return c2v_v2c_target_weight - - def get_targets(self, gt_bboxes, gt_labels, feat_shape, img_shape): + def get_targets(self, gt_bboxes, gt_labels, feat_shape, img_shape, c2v_pred, v2c_pred): """Compute regression and classification targets in multiple images. Args: @@ -442,17 +278,12 @@ def get_targets(self, gt_bboxes, gt_labels, feat_shape, img_shape): width_ratio = float(feat_w / img_w) height_ratio = float(feat_h / img_h) - center_heatmap_target = gt_bboxes[-1].new_zeros( - [bs, 2 * self.num_classes, feat_h, feat_w] - ) - # wh_target = gt_bboxes[-1].new_zeros([bs, 2, feat_h, feat_w]) + center_heatmap_target = gt_bboxes[-1].new_zeros([bs, 2 * self.num_classes, feat_h, feat_w]) offset_target = gt_bboxes[-1].new_zeros([bs, 2, feat_h, feat_w]) - wh_offset_target_weight = gt_bboxes[-1].new_zeros( - [bs, 2, feat_h, feat_w] - ) + offset_target_weight = gt_bboxes[-1].new_zeros([bs, 2, feat_h, feat_w]) c2v_target = gt_bboxes[-1].new_zeros([bs, 8, feat_h, feat_w]) v2c_target = gt_bboxes[-1].new_zeros([bs, 8, feat_h, feat_w]) - c2v_v2c_target_weight = gt_bboxes[-1].new_zeros([bs, 8, feat_h, feat_w]) + pairing_weight = gt_bboxes[-1].new_zeros([bs, 8, feat_h, feat_w]) for batch_id in range(bs): gt_bbox = gt_bboxes[batch_id] @@ -466,9 +297,7 @@ def get_targets(self, gt_bboxes, gt_labels, feat_shape, img_shape): ctx, cty = ct scale_box_h = (gt_bbox[j][3] - gt_bbox[j][1]) * height_ratio scale_box_w = (gt_bbox[j][2] - gt_bbox[j][0]) * width_ratio - radius = gaussian_radius( - [scale_box_h, scale_box_w], min_overlap=0.3 - ) + radius = gaussian_radius([scale_box_h, scale_box_w], min_overlap=0.3) radius = max(0, int(radius)) ind = gt_label[j] gen_gaussian_target( @@ -527,9 +356,6 @@ def get_targets(self, gt_bboxes, gt_labels, feat_shape, img_shape): radius, ) - # wh_target[batch_id, 0, cty_int, ctx_int] = scale_box_w - # wh_target[batch_id, 1, cty_int, ctx_int] = scale_box_h - offset_target[batch_id, 0, cty_int, ctx_int] = ctx - ctx_int offset_target[batch_id, 0, tl_y_int, tl_x_int] = tl_x - tl_x_int offset_target[batch_id, 0, tr_y_int, tr_x_int] = tr_x - tr_x_int @@ -542,7 +368,11 @@ def get_targets(self, gt_bboxes, gt_labels, feat_shape, img_shape): offset_target[batch_id, 1, br_y_int, br_x_int] = br_y - br_y_int offset_target[batch_id, 1, bl_y_int, bl_x_int] = bl_y - bl_y_int - wh_offset_target_weight[batch_id, :, cty_int, ctx_int] = 1 + offset_target_weight[batch_id, :, cty_int, ctx_int] = 1 + offset_target_weight[batch_id, :, tl_y_int, tl_x_int] = 1 + offset_target_weight[batch_id, :, tr_y_int, tr_x_int] = 1 + offset_target_weight[batch_id, :, br_y_int, br_x_int] = 1 + offset_target_weight[batch_id, :, bl_y_int, bl_x_int] = 1 c2v_target[batch_id, 0, cty_int, ctx_int] = -scale_box_w / 2 c2v_target[batch_id, 1, cty_int, ctx_int] = -scale_box_h / 2 @@ -562,21 +392,47 @@ def get_targets(self, gt_bboxes, gt_labels, feat_shape, img_shape): v2c_target[batch_id, 6, bl_y_int, bl_x_int] = scale_box_w / 2 v2c_target[batch_id, 7, bl_y_int, bl_x_int] = -scale_box_h / 2 - c2v_v2c_target_weight[batch_id, :, cty_int, ctx_int] = 1 - c2v_v2c_target_weight[batch_id, :, tl_y_int, tl_x_int] = 1 - c2v_v2c_target_weight[batch_id, :, tr_y_int, tr_x_int] = 1 - c2v_v2c_target_weight[batch_id, :, br_y_int, br_x_int] = 1 - c2v_v2c_target_weight[batch_id, :, bl_y_int, bl_x_int] = 1 + # pairing_weight[batch_id, :, cty_int, ctx_int] = 1 + # pairing_weight[batch_id, :, tl_y_int, tl_x_int] = 1 + # pairing_weight[batch_id, :, tr_y_int, tr_x_int] = 1 + # pairing_weight[batch_id, :, br_y_int, br_x_int] = 1 + # pairing_weight[batch_id, :, bl_y_int, bl_x_int] = 1 + + # Pairing loss + for idx, v_x_int, v_y_int in ( + (0, tl_x_int, tl_y_int), + (1, tr_x_int, tr_y_int), + (2, br_x_int, br_y_int), + (3, bl_x_int, bl_y_int), + ): + for k in range(2): + D_cv = min( + torch.tensor(1.0), + ( + torch.abs( + c2v_pred[batch_id, 2 * idx + k, cty_int, ctx_int] + - c2v_target[batch_id, 2 * idx + k, cty_int, ctx_int] + ) + + torch.abs( + v2c_pred[batch_id, 2 * idx + k, v_y_int, v_x_int] + - v2c_target[batch_id, 2 * idx + k, v_y_int, v_x_int] + ) + ) + / torch.abs(c2v_target[batch_id, 2 * idx + k, cty_int, ctx_int]), + ) + w = 1 - torch.exp(-pi * D_cv) + pairing_weight[batch_id, 2 * idx + k, cty_int, ctx_int] += w + pairing_weight[batch_id, 2 * idx + k, v_y_int, v_x_int] += w avg_factor = max(1, center_heatmap_target.eq(1).sum()) target_result = dict( center_heatmap_target=center_heatmap_target, # wh_target=wh_target, offset_target=offset_target, - wh_offset_target_weight=wh_offset_target_weight, + offset_target_weight=offset_target_weight, center2vertex_target=c2v_target, vertex2center_target=v2c_target, - c2v_v2c_target_weight=c2v_v2c_target_weight, + pairing_weight=pairing_weight, ) return target_result, avg_factor @@ -666,9 +522,7 @@ def get_bboxes( for img_id in range(len(img_metas)): result_list.append( self._get_bboxes_single( - center_heatmap_preds[0][ - img_id : img_id + 1, 0:1, ... - ], ### 000!!!! + center_heatmap_preds[0][img_id : img_id + 1, 0:1, ...], ### 000!!!! wh_preds[0][img_id : img_id + 1, ...], offset_preds[0][img_id : img_id + 1, ...], img_metas[img_id], @@ -726,20 +580,14 @@ def _get_bboxes_single( det_bboxes = batch_det_bboxes.view([-1, 5]) det_labels = batch_labels.view(-1) - batch_border = det_bboxes.new_tensor(img_meta["border"])[ - ..., [2, 0, 2, 0] - ] + batch_border = det_bboxes.new_tensor(img_meta["border"])[..., [2, 0, 2, 0]] det_bboxes[..., :4] -= batch_border if rescale: - det_bboxes[..., :4] /= det_bboxes.new_tensor( - img_meta["scale_factor"] - ) + det_bboxes[..., :4] /= det_bboxes.new_tensor(img_meta["scale_factor"]) if with_nms: - det_bboxes, det_labels = self._bboxes_nms( - det_bboxes, det_labels, self.test_cfg - ) + det_bboxes, det_labels = self._bboxes_nms(det_bboxes, det_labels, self.test_cfg) return det_bboxes, det_labels def decode_heatmap( @@ -774,13 +622,9 @@ def decode_heatmap( height, width = center_heatmap_pred.shape[2:] inp_h, inp_w = img_shape - center_heatmap_pred = get_local_maximum( - center_heatmap_pred, kernel=kernel - ) + center_heatmap_pred = get_local_maximum(center_heatmap_pred, kernel=kernel) - *batch_dets, topk_ys, topk_xs = get_topk_from_heatmap( - center_heatmap_pred, k=k - ) + *batch_dets, topk_ys, topk_xs = get_topk_from_heatmap(center_heatmap_pred, k=k) batch_scores, batch_index, batch_topk_labels = batch_dets wh = transpose_and_gather_feat(wh_pred, batch_index) @@ -793,17 +637,13 @@ def decode_heatmap( br_y = (topk_ys + wh[..., 1] / 2) * (inp_h / height) batch_bboxes = torch.stack([tl_x, tl_y, br_x, br_y], dim=2) - batch_bboxes = torch.cat( - (batch_bboxes, batch_scores[..., None]), dim=-1 - ) + batch_bboxes = torch.cat((batch_bboxes, batch_scores[..., None]), dim=-1) return batch_bboxes, batch_topk_labels def _bboxes_nms(self, bboxes, labels, cfg): if labels.numel() > 0: max_num = cfg.max_per_img - bboxes, keep = batched_nms( - bboxes[:, :4], bboxes[:, -1].contiguous(), labels, cfg.nms - ) + bboxes, keep = batched_nms(bboxes[:, :4], bboxes[:, -1].contiguous(), labels, cfg.nms) if max_num > 0: bboxes = bboxes[:max_num] labels = labels[keep][:max_num] From 4069ca95bc99a36919a258beae3d8a16e9732a01 Mon Sep 17 00:00:00 2001 From: ArchieAlexArkhipov Date: Thu, 11 May 2023 18:25:34 +0000 Subject: [PATCH 09/25] fix wrong heigth of eval --- mmdet/models/dense_heads/cycle_centernet_head.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mmdet/models/dense_heads/cycle_centernet_head.py b/mmdet/models/dense_heads/cycle_centernet_head.py index 50771317..c862fb5f 100644 --- a/mmdet/models/dense_heads/cycle_centernet_head.py +++ b/mmdet/models/dense_heads/cycle_centernet_head.py @@ -486,7 +486,6 @@ def get_bboxes( """ assert ( len(center_heatmap_preds) - # == len(wh_preds) == len(offset_preds) == len(center2vertex_preds) == len(vertex2center_preds) @@ -516,7 +515,7 @@ def get_bboxes( -center2vertex_preds[0][:, 1, ...] - center2vertex_preds[0][:, 3, ...] + center2vertex_preds[0][:, 5, ...] - - center2vertex_preds[0][:, 7, ...] + + center2vertex_preds[0][:, 7, ...] ) / 2 result_list = [] for img_id in range(len(img_metas)): From 95212fbac6753a8724cb24525e282f2fd24796be Mon Sep 17 00:00:00 2001 From: ArchieAlexArkhipov Date: Fri, 12 May 2023 23:25:02 +0000 Subject: [PATCH 10/25] fix heatmap one chanel --- .../dense_heads/cycle_centernet_head.py | 152 +++++++----------- 1 file changed, 59 insertions(+), 93 deletions(-) diff --git a/mmdet/models/dense_heads/cycle_centernet_head.py b/mmdet/models/dense_heads/cycle_centernet_head.py index c862fb5f..08f7353d 100644 --- a/mmdet/models/dense_heads/cycle_centernet_head.py +++ b/mmdet/models/dense_heads/cycle_centernet_head.py @@ -46,11 +46,9 @@ def __init__( feat_channel, num_classes, loss_center_heatmap=dict(type="GaussianFocalLoss", loss_weight=1.0), - # loss_wh=dict(type="L1Loss", loss_weight=0.1), loss_offset=dict(type="L1Loss", loss_weight=1.0), loss_c2v=dict(type="L1Loss", loss_weight=1.0), loss_v2c=dict(type="L1Loss", loss_weight=0.5), - # loss_pairing=dict(type="PairingLoss", loss_weight=1.0), train_cfg=None, test_cfg=None, init_cfg=None, @@ -58,15 +56,12 @@ def __init__( super(CycleCenterNetHeadL1, self).__init__(init_cfg) self.num_classes = num_classes self.heatmap_head = self._build_head(in_channel, feat_channel, 2 * num_classes) - # self.wh_head = self._build_head(in_channel, feat_channel, 2) self.offset_head = self._build_head(in_channel, feat_channel, 2) self.center2vertex_head = self._build_head(in_channel, feat_channel, 8) self.vertex2center_head = self._build_head(in_channel, feat_channel, 8) self.loss_center_heatmap = build_loss(loss_center_heatmap) - # self.loss_wh = build_loss(loss_wh) self.loss_offset = build_loss(loss_offset) - # self.loss_pairing = build_loss(loss_pairing) self.loss_c2v = build_loss(loss_c2v) self.loss_v2c = build_loss(loss_v2c) @@ -88,7 +83,6 @@ def init_weights(self): bias_init = bias_init_with_prob(0.1) self.heatmap_head[-1].bias.data.fill_(bias_init) for head in [ - # self.wh_head, self.offset_head, self.center2vertex_head, self.vertex2center_head, @@ -127,13 +121,11 @@ def forward_single(self, feat): offset_pred (Tensor): offset predicts, the channels number is 2. """ center_heatmap_pred = self.heatmap_head(feat).sigmoid() - # wh_pred = self.wh_head(feat) offset_pred = self.offset_head(feat) center2vertex_pred = self.center2vertex_head(feat) vertex2center_pred = self.vertex2center_head(feat) return ( center_heatmap_pred, - # wh_pred, offset_pred, center2vertex_pred, vertex2center_pred, @@ -187,7 +179,6 @@ def loss( """ assert ( len(center_heatmap_preds) - # == len(wh_preds) == len(offset_preds) == len(center2vertex_pred) == len(vertex2center_pred) @@ -208,7 +199,6 @@ def loss( ) center_heatmap_target = target_result["center_heatmap_target"] - # wh_target = target_result["wh_target"] offset_target = target_result["offset_target"] offset_target_weight = target_result["offset_target_weight"] center2vertex_target = target_result["center2vertex_target"] @@ -287,74 +277,53 @@ def get_targets(self, gt_bboxes, gt_labels, feat_shape, img_shape, c2v_pred, v2c for batch_id in range(bs): gt_bbox = gt_bboxes[batch_id] - gt_label = gt_labels[batch_id] center_x = (gt_bbox[:, [0]] + gt_bbox[:, [2]]) * width_ratio / 2 center_y = (gt_bbox[:, [1]] + gt_bbox[:, [3]]) * height_ratio / 2 gt_centers = torch.cat((center_x, center_y), dim=1) - + vertexes = {} for j, ct in enumerate(gt_centers): ctx_int, cty_int = ct.int() ctx, cty = ct scale_box_h = (gt_bbox[j][3] - gt_bbox[j][1]) * height_ratio scale_box_w = (gt_bbox[j][2] - gt_bbox[j][0]) * width_ratio + scale_box_h_r = scale_box_h / 2 + scale_box_w_r = scale_box_w / 2 radius = gaussian_radius([scale_box_h, scale_box_w], min_overlap=0.3) radius = max(0, int(radius)) - ind = gt_label[j] gen_gaussian_target( - center_heatmap_target[batch_id, ind], + center_heatmap_target[batch_id, 0], [ctx_int, cty_int], radius, ) - tl_x, tl_y = ( - gt_bbox[j][0] * width_ratio, - gt_bbox[j][1] * height_ratio, + tl_x, tr_x, br_x, bl_x = map( + lambda x: x * width_ratio, (gt_bbox[j][0], gt_bbox[j][2], gt_bbox[j][2], gt_bbox[j][0]) ) - tr_x, tr_y = ( - gt_bbox[j][2] * width_ratio, - gt_bbox[j][1] * height_ratio, + tl_y, tr_y, br_y, bl_y = map( + lambda x: x * height_ratio, (gt_bbox[j][1], gt_bbox[j][1], gt_bbox[j][3], gt_bbox[j][3]) ) - br_x, br_y = ( - gt_bbox[j][2] * width_ratio, - gt_bbox[j][3] * height_ratio, + tl_x_int, tl_y_int, tr_x_int, tr_y_int, br_x_int, br_y_int, bl_x_int, bl_y_int = map( + int, (tl_x, tl_y, tr_x, tr_y, br_x, br_y, bl_x, bl_y) ) - bl_x, bl_y = ( - gt_bbox[j][0] * width_ratio, - gt_bbox[j][3] * height_ratio, + tl_x_int, tr_x_int, br_x_int, bl_x_int = map( + lambda x: feat_w - 1 if x >= feat_w else x if x >= 0 else 0, + (tl_x_int, tr_x_int, br_x_int, bl_x_int), ) - - tl_x_int, tl_y_int = int(tl_x), int(tl_y) - tr_x_int, tr_y_int = int(tr_x), int(tr_y) - br_x_int, br_y_int = int(br_x), int(br_y) - bl_x_int, bl_y_int = int(bl_x), int(bl_y) - - tl_x_int = tl_x_int if 0 <= tl_x_int else 0 - tr_x_int = tr_x_int if 0 <= tr_x_int else 0 - br_x_int = br_x_int if 0 <= br_x_int else 0 - bl_x_int = bl_x_int if 0 <= bl_x_int else 0 - tl_x_int = tl_x_int if tl_x_int < feat_w else feat_w - 1 - tr_x_int = tr_x_int if tr_x_int < feat_w else feat_w - 1 - br_x_int = br_x_int if br_x_int < feat_w else feat_w - 1 - bl_x_int = bl_x_int if bl_x_int < feat_w else feat_w - 1 - tl_y_int = tl_y_int if 0 <= tl_y_int else 0 - tr_y_int = tr_y_int if 0 <= tr_y_int else 0 - br_y_int = br_y_int if 0 <= br_y_int else 0 - bl_y_int = bl_y_int if 0 <= bl_y_int else 0 - tl_y_int = tl_y_int if tl_y_int < feat_h else feat_h - 1 - tr_y_int = tr_y_int if tr_y_int < feat_h else feat_h - 1 - br_y_int = br_y_int if br_y_int < feat_h else feat_h - 1 - bl_y_int = bl_y_int if bl_y_int < feat_h else feat_h - 1 + tl_y_int, tr_y_int, br_y_int, bl_y_int = map( + lambda x: feat_h - 1 if x >= feat_h else x if x >= 0 else 0, + (tl_y_int, tr_y_int, br_y_int, bl_y_int), + ) + vtx_gaus_r = max(0, int(gaussian_radius([scale_box_h_r, scale_box_w_r], min_overlap=0.3))) for x, y in ( (tl_x_int, tl_y_int), (tr_x_int, tr_y_int), (br_x_int, br_y_int), (bl_x_int, bl_y_int), ): - gen_gaussian_target( - center_heatmap_target[batch_id, 2 * ind], - [x, y], - radius, - ) + if (x, y) in vertexes: + vertexes[(x, y)].append(vtx_gaus_r) + else: + vertexes[(x, y)] = [vtx_gaus_r] offset_target[batch_id, 0, cty_int, ctx_int] = ctx - ctx_int offset_target[batch_id, 0, tl_y_int, tl_x_int] = tl_x - tl_x_int @@ -374,29 +343,23 @@ def get_targets(self, gt_bboxes, gt_labels, feat_shape, img_shape, c2v_pred, v2c offset_target_weight[batch_id, :, br_y_int, br_x_int] = 1 offset_target_weight[batch_id, :, bl_y_int, bl_x_int] = 1 - c2v_target[batch_id, 0, cty_int, ctx_int] = -scale_box_w / 2 - c2v_target[batch_id, 1, cty_int, ctx_int] = -scale_box_h / 2 - c2v_target[batch_id, 2, cty_int, ctx_int] = scale_box_w / 2 - c2v_target[batch_id, 3, cty_int, ctx_int] = -scale_box_h / 2 - c2v_target[batch_id, 4, cty_int, ctx_int] = scale_box_w / 2 - c2v_target[batch_id, 5, cty_int, ctx_int] = scale_box_h / 2 - c2v_target[batch_id, 6, cty_int, ctx_int] = -scale_box_w / 2 - c2v_target[batch_id, 7, cty_int, ctx_int] = scale_box_h / 2 - - v2c_target[batch_id, 0, tl_y_int, tl_x_int] = scale_box_w / 2 - v2c_target[batch_id, 1, tl_y_int, tl_x_int] = scale_box_h / 2 - v2c_target[batch_id, 2, tr_y_int, tr_x_int] = -scale_box_w / 2 - v2c_target[batch_id, 3, tr_y_int, tr_x_int] = scale_box_h / 2 - v2c_target[batch_id, 4, br_y_int, br_x_int] = -scale_box_w / 2 - v2c_target[batch_id, 5, br_y_int, br_x_int] = -scale_box_h / 2 - v2c_target[batch_id, 6, bl_y_int, bl_x_int] = scale_box_w / 2 - v2c_target[batch_id, 7, bl_y_int, bl_x_int] = -scale_box_h / 2 - - # pairing_weight[batch_id, :, cty_int, ctx_int] = 1 - # pairing_weight[batch_id, :, tl_y_int, tl_x_int] = 1 - # pairing_weight[batch_id, :, tr_y_int, tr_x_int] = 1 - # pairing_weight[batch_id, :, br_y_int, br_x_int] = 1 - # pairing_weight[batch_id, :, bl_y_int, bl_x_int] = 1 + c2v_target[batch_id, 0, cty_int, ctx_int] = -scale_box_w_r + c2v_target[batch_id, 1, cty_int, ctx_int] = -scale_box_h_r + c2v_target[batch_id, 2, cty_int, ctx_int] = scale_box_w_r + c2v_target[batch_id, 3, cty_int, ctx_int] = -scale_box_h_r + c2v_target[batch_id, 4, cty_int, ctx_int] = scale_box_w_r + c2v_target[batch_id, 5, cty_int, ctx_int] = scale_box_h_r + c2v_target[batch_id, 6, cty_int, ctx_int] = -scale_box_w_r + c2v_target[batch_id, 7, cty_int, ctx_int] = scale_box_h_r + + v2c_target[batch_id, 0, tl_y_int, tl_x_int] = scale_box_w_r + v2c_target[batch_id, 1, tl_y_int, tl_x_int] = scale_box_h_r + v2c_target[batch_id, 2, tr_y_int, tr_x_int] = -scale_box_w_r + v2c_target[batch_id, 3, tr_y_int, tr_x_int] = scale_box_h_r + v2c_target[batch_id, 4, br_y_int, br_x_int] = -scale_box_w_r + v2c_target[batch_id, 5, br_y_int, br_x_int] = -scale_box_h_r + v2c_target[batch_id, 6, bl_y_int, bl_x_int] = scale_box_w_r + v2c_target[batch_id, 7, bl_y_int, bl_x_int] = -scale_box_h_r # Pairing loss for idx, v_x_int, v_y_int in ( @@ -423,11 +386,16 @@ def get_targets(self, gt_bboxes, gt_labels, feat_shape, img_shape, c2v_pred, v2c w = 1 - torch.exp(-pi * D_cv) pairing_weight[batch_id, 2 * idx + k, cty_int, ctx_int] += w pairing_weight[batch_id, 2 * idx + k, v_y_int, v_x_int] += w + for point, radiuses in vertexes.items(): + gen_gaussian_target( + heatmap=center_heatmap_target[batch_id, 1], + center=list(point), + radius=max(0, int(sum(radiuses) / len(radiuses))), + ) avg_factor = max(1, center_heatmap_target.eq(1).sum()) target_result = dict( center_heatmap_target=center_heatmap_target, - # wh_target=wh_target, offset_target=offset_target, offset_target_weight=offset_target_weight, center2vertex_target=c2v_target, @@ -439,7 +407,6 @@ def get_targets(self, gt_bboxes, gt_labels, feat_shape, img_shape, c2v_pred, v2c @force_fp32( apply_to=( "center_heatmap_preds", - # "wh_preds", "offset_preds", "center2vertex_preds", "vertex2center_preds", @@ -448,7 +415,6 @@ def get_targets(self, gt_bboxes, gt_labels, feat_shape, img_shape, c2v_pred, v2c def get_bboxes( self, center_heatmap_preds, - # wh_preds, offset_preds, center2vertex_preds, vertex2center_preds, @@ -492,19 +458,7 @@ def get_bboxes( == 1 ) wh_preds = [torch.zeros_like(offset_preds[0])] - # diff = (center2vertex_preds[0] - vertex2center_preds[0]) / 2 - # wh_preds[0][:, 0, ...] = ( - # -diff[:, 0, ...] - # + diff[:, 2, ...] - # + diff[:, 4, ...] - # - diff[:, 6, ...] - # ) / 2 - # wh_preds[0][:, 1, ...] = ( - # -diff[:, 1, ...] - # - diff[:, 3, ...] - # + diff[:, 5, ...] - # - diff[:, 7, ...] - # ) / 2 + # wh_preds_vc = [torch.ones_like(offset_preds[0]) * 5] wh_preds[0][:, 0, ...] = ( -center2vertex_preds[0][:, 0, ...] + center2vertex_preds[0][:, 2, ...] @@ -521,7 +475,7 @@ def get_bboxes( for img_id in range(len(img_metas)): result_list.append( self._get_bboxes_single( - center_heatmap_preds[0][img_id : img_id + 1, 0:1, ...], ### 000!!!! + center_heatmap_preds[0][img_id : img_id + 1, 0:1, ...], wh_preds[0][img_id : img_id + 1, ...], offset_preds[0][img_id : img_id + 1, ...], img_metas[img_id], @@ -529,6 +483,18 @@ def get_bboxes( with_nms=with_nms, ) ) + # for img_id in range(len(img_metas)): + # result_list.append( + # self._get_bboxes_single( + # center_heatmap_preds[0][img_id : img_id + 1, 1:2, ...], + # wh_preds_vc[0][img_id : img_id + 1, ...], + # offset_preds[0][img_id : img_id + 1, ...], + # img_metas[img_id], + # rescale=rescale, + # with_nms=with_nms, + # ) + # ) + # ) return result_list def _get_bboxes_single( From 617eea2f063ea8e9ca4f8019c7c5baff642f1cbe Mon Sep 17 00:00:00 2001 From: ArchieAlexArkhipov Date: Mon, 15 May 2023 12:09:12 +0000 Subject: [PATCH 11/25] upd doc --- mmdet/models/dense_heads/cycle_centernet_head.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mmdet/models/dense_heads/cycle_centernet_head.py b/mmdet/models/dense_heads/cycle_centernet_head.py index 08f7353d..3f016171 100644 --- a/mmdet/models/dense_heads/cycle_centernet_head.py +++ b/mmdet/models/dense_heads/cycle_centernet_head.py @@ -237,7 +237,8 @@ def loss( ) def get_targets(self, gt_bboxes, gt_labels, feat_shape, img_shape, c2v_pred, v2c_pred): - """Compute regression and classification targets in multiple images. + """Compute regression and classification targets in multiple images and + compute weight for Pairing Loss. Args: gt_bboxes (list[Tensor]): Ground truth bboxes for each image with @@ -245,7 +246,9 @@ def get_targets(self, gt_bboxes, gt_labels, feat_shape, img_shape, c2v_pred, v2c gt_labels (list[Tensor]): class indices corresponding to each box. feat_shape (list[int]): feature map shape with value [B, _, H, W] img_shape (list[int]): image shape in [h, w] format. - + c2v_pred (list[Tensor]): center2vertex branch shape with value [B, 8, H, W] + v2c_pred (list[Tensor]): vertex2center branch shape with value [B, 8, H, W] + Returns: tuple[dict,float]: The float value is mean avg_factor, the dict has components below: From 9c0b221b2db20f9c040753d90c1dcb90fcb39d49 Mon Sep 17 00:00:00 2001 From: ArchieAlexArkhipov Date: Mon, 15 May 2023 13:44:53 +0000 Subject: [PATCH 12/25] del custom dla --- mmdet/models/backbones/__init__.py | 2 - mmdet/models/backbones/dla_custom.py | 634 --------------------------- 2 files changed, 636 deletions(-) delete mode 100644 mmdet/models/backbones/dla_custom.py diff --git a/mmdet/models/backbones/__init__.py b/mmdet/models/backbones/__init__.py index 09cd7722..a66650a3 100644 --- a/mmdet/models/backbones/__init__.py +++ b/mmdet/models/backbones/__init__.py @@ -3,7 +3,6 @@ from .darknet import Darknet from .detectors_resnet import DetectoRS_ResNet from .detectors_resnext import DetectoRS_ResNeXt -from .dla_custom import DLANetCustom from .dla_mmdet3d import DLANetMMDet3D from .efficientnet import EfficientNet from .hourglass import HourglassNet @@ -39,6 +38,5 @@ "PyramidVisionTransformer", "PyramidVisionTransformerV2", "EfficientNet", - "DLANetCustom", "DLANetMMDet3D", ] diff --git a/mmdet/models/backbones/dla_custom.py b/mmdet/models/backbones/dla_custom.py deleted file mode 100644 index ebf8a12a..00000000 --- a/mmdet/models/backbones/dla_custom.py +++ /dev/null @@ -1,634 +0,0 @@ -import torch -import torch.utils.checkpoint as cp -from mmcv.cnn import build_conv_layer, build_norm_layer, constant_init -from mmcv.runner import load_checkpoint -from mmdet.utils import get_root_logger -from torch import nn -from torch.nn.modules.batchnorm import _BatchNorm - -from ..builder import BACKBONES -from .resnet import BasicBlock as _BasicBlock -from .resnet import Bottleneck as _Bottleneck -from .resnet import ResNet - - -class BasicBlock(_BasicBlock): - def __init__( - self, - inplanes, - planes, - stride=1, - dilation=1, - downsample=None, - style="pytorch", - with_cp=False, - conv_cfg=None, - norm_cfg=dict(type="BN"), - dcn=None, - plugins=None, - ): - super(BasicBlock, self).__init__( - inplanes, - planes, - stride=stride, - dilation=dilation, - downsample=downsample, - style=style, - with_cp=with_cp, - conv_cfg=conv_cfg, - norm_cfg=norm_cfg, - dcn=dcn, - plugins=plugins, - ) - - def forward(self, x, identity=None): - """Forward function.""" - - def _inner_forward(x, identity): - - if identity is None: - identity = x - - out = self.conv1(x) - out = self.norm1(out) - out = self.relu(out) - - out = self.conv2(out) - out = self.norm2(out) - - if self.downsample is not None: - identity = self.downsample(x) - - out += identity - - return out - - if self.with_cp and x.requires_grad: - out = cp.checkpoint(_inner_forward, x, identity) - else: - out = _inner_forward(x, identity) - - out = self.relu(out) - - return out - - -class Bottleneck(_Bottleneck): - expansion = 2 - - def __init__( - self, - inplanes, - planes, - stride=1, - dilation=1, - downsample=None, - style="pytorch", - with_cp=False, - conv_cfg=None, - norm_cfg=dict(type="BN"), - dcn=None, - plugins=None, - ): - super(Bottleneck, self).__init__( - inplanes, - planes, - stride=stride, - dilation=dilation, - downsample=downsample, - style=style, - with_cp=with_cp, - conv_cfg=conv_cfg, - norm_cfg=norm_cfg, - dcn=dcn, - plugins=plugins, - ) - """Bottleneck block for DLA.""" - - expansion = self.expansion - bottle_planes = planes // expansion - - self.norm1_name, norm1 = build_norm_layer( - norm_cfg, bottle_planes, postfix=1 - ) - self.norm2_name, norm2 = build_norm_layer( - norm_cfg, bottle_planes, postfix=2 - ) - self.norm3_name, norm3 = build_norm_layer(norm_cfg, planes, postfix=3) - - self.conv1 = build_conv_layer( - conv_cfg, - inplanes, - bottle_planes, - kernel_size=1, - stride=self.conv1_stride, - bias=False, - ) - - self.add_module(self.norm1_name, norm1) - fallback_on_stride = False - if self.with_dcn: - fallback_on_stride = dcn.pop("fallback_on_stride", False) - if not self.with_dcn or fallback_on_stride: - self.conv2 = build_conv_layer( - conv_cfg, - bottle_planes, - bottle_planes, - kernel_size=3, - stride=self.conv2_stride, - padding=dilation, - dilation=dilation, - bias=False, - ) - else: - assert self.conv_cfg is None, "conv_cfg must be None for DCN" - self.conv2 = build_conv_layer( - dcn, - bottle_planes, - bottle_planes, - kernel_size=3, - stride=self.conv2_stride, - padding=dilation, - dilation=dilation, - bias=False, - ) - self.add_module(self.norm2_name, norm2) - self.conv3 = build_conv_layer( - conv_cfg, bottle_planes, planes, kernel_size=1, bias=False - ) - self.add_module(self.norm3_name, norm3) - - if self.with_plugins: - for name in self.after_conv1_plugin_names: - delattr(self, name) - for name in self.after_conv2_plugin_names: - delattr(self, name) - for name in self.after_conv3_plugin_names: - delattr(self, name) - self.after_conv1_plugin_names = self.make_block_plugins( - bottle_planes, self.after_conv1_plugins - ) - self.after_conv2_plugin_names = self.make_block_plugins( - bottle_planes, self.after_conv2_plugins - ) - self.after_conv3_plugin_names = self.make_block_plugins( - planes, self.after_conv3_plugins - ) - - def forward(self, x, identity=None): - """Forward function.""" - - def _inner_forward(x, identity): - if identity is None: - identity = x - - out = self.conv1(x) - out = self.norm1(out) - out = self.relu(out) - - if self.with_plugins: - out = self.forward_plugin(out, self.after_conv1_plugin_names) - - out = self.conv2(out) - out = self.norm2(out) - out = self.relu(out) - - if self.with_plugins: - out = self.forward_plugin(out, self.after_conv2_plugin_names) - - out = self.conv3(out) - out = self.norm3(out) - - if self.with_plugins: - out = self.forward_plugin(out, self.after_conv3_plugin_names) - - if self.downsample is not None: - identity = self.downsample(x) - - out += identity - - return out - - if self.with_cp and x.requires_grad: - out = cp.checkpoint(_inner_forward, x, identity) - else: - out = _inner_forward(x, identity) - - out = self.relu(out) - - return out - - -class Root(nn.Module): - def __init__( - self, - in_channels, - out_channels, - kernel_size, - residual, - conv_cfg=None, - norm_cfg=dict(type="BN"), - ): - super(Root, self).__init__() - self.conv = build_conv_layer( - conv_cfg, - in_channels, - out_channels, - kernel_size, - stride=1, - bias=False, - padding=(kernel_size - 1) // 2, - ) - self.norm_name, norm = build_norm_layer(norm_cfg, out_channels) - self.add_module(self.norm_name, norm) - self.relu = nn.ReLU(inplace=True) - self.residual = residual - - @property - def norm(self): - """nn.Module: the normalization layer named "norm" """ - return getattr(self, self.norm_name) - - def forward(self, *x): - children = x - x = self.conv(torch.cat(x, 1)) - x = self.norm(x) - if self.residual: - x += children[0] - x = self.relu(x) - - return x - - -class Tree(nn.Module): - def __init__( - self, - levels, - block, - in_channels, - out_channels, - stride=1, - level_root=False, - root_dim=0, - root_kernel_size=1, - dilation=1, - root_residual=False, - conv_cfg=None, - norm_cfg=dict(type="BN"), - with_cp=False, - dcn=None, - plugins=None, - style="pytorch", - ): - super(Tree, self).__init__() - if root_dim == 0: - root_dim = 2 * out_channels - if level_root: - root_dim += in_channels - if levels == 1: - self.tree1 = block( - in_channels, - out_channels, - stride, - dilation=dilation, - conv_cfg=conv_cfg, - norm_cfg=norm_cfg, - with_cp=with_cp, - dcn=dcn, - plugins=plugins, - style=style, - ) - self.tree2 = block( - out_channels, - out_channels, - 1, - dilation=dilation, - conv_cfg=conv_cfg, - norm_cfg=norm_cfg, - with_cp=with_cp, - dcn=dcn, - plugins=plugins, - style=style, - ) - self.root = Root( - root_dim, - out_channels, - root_kernel_size, - root_residual, - conv_cfg=conv_cfg, - norm_cfg=norm_cfg, - ) - else: - self.tree1 = Tree( - levels - 1, - block, - in_channels, - out_channels, - stride, - root_dim=0, - root_kernel_size=root_kernel_size, - dilation=dilation, - root_residual=root_residual, - conv_cfg=conv_cfg, - norm_cfg=norm_cfg, - with_cp=with_cp, - dcn=dcn, - plugins=plugins, - style=style, - ) - self.tree2 = Tree( - levels - 1, - block, - out_channels, - out_channels, - root_dim=root_dim + out_channels, - root_kernel_size=root_kernel_size, - dilation=dilation, - root_residual=root_residual, - conv_cfg=conv_cfg, - norm_cfg=norm_cfg, - with_cp=with_cp, - dcn=dcn, - plugins=plugins, - style=style, - ) - - self.level_root = level_root - self.downsample = None - self.project = None - self.levels = levels - if stride > 1: - self.downsample = nn.MaxPool2d(stride, stride=stride) - if in_channels != out_channels and self.levels == 1: - self.project = nn.Sequential( - build_conv_layer( - conv_cfg, - in_channels, - out_channels, - kernel_size=1, - stride=1, - bias=False, - ), - build_norm_layer(norm_cfg, out_channels)[1], - ) - - def forward(self, x, residual=None, children=None): - children = [] if children is None else children - bottom = self.downsample(x) if self.downsample else x - residual = self.project(bottom) if self.project else bottom - if self.level_root: - children.append(bottom) - x1 = self.tree1(x, residual) - if self.levels == 1: - x2 = self.tree2(x1) - x = self.root(x2, x1, *children) - else: - children.append(x1) - x = self.tree2(x1, children=children) - return x - - -@BACKBONES.register_module() -class DLANetCustom(nn.Module): - """DLA backbone. - - Args: - depth (int): Depth of dla, from {34, 46, 60, 102, 169}. - in_channels (int): Number of input image channels. Default: 3. - num_stages (int): dla stages. Default: 4. - out_indices (Sequence[int]): Output from which stages. - strides (Sequence[int]): Strides of the first block of each stage. - style (str): `pytorch` or `caffe`. If set to "pytorch", the stride-two - layer is the 3x3 conv layer, otherwise the stride-two layer is - the first 1x1 conv layer. - frozen_stages (int): Stages to be frozen (stop grad and set eval mode). - -1 means not freezing any parameters. - conv_cfg (dict): Dictionary to construct and config convolution layer. - norm_cfg (dict): Dictionary to construct and config norm layer. - norm_eval (bool): Whether to set norm layers to eval mode, namely, - freeze running stats (mean and var). Note: Effect on Batch Norm - and its variants only. - dcn (dict): Dictionary to construct and config deformable convolution - layer - stage_with_dcn (tuple[bool]): Stages to apply dcn, length - should be same as 'num_stages'. - plugins (list[dict]): List of plugins for stages, each dict contains: - - - cfg (dict, required): Cfg dict to build plugin. - - position (str, required): Position inside block to insert - plugin, options are 'after_conv1', 'after_conv2', 'after_conv3'. - - stages (tuple[bool], optional): Stages to apply plugin, length - should be same as 'num_stages'. - with_cp (bool): Use checkpoint or not. Using checkpoint will save some - memory while slowing down the training speed. - zero_init_residual (bool): Whether to use zero init for last norm layer - in resblocks to let them behave as identity. - stage_with_level_root (tuple[bool]): Stages to apply level_root, length - should be same as 'num_stages'. - residual_root (bool): whether to use residual in root layer - - Example: - >>> from mmdet.models import DLANet - >>> import torch - >>> self = DLANet(depth=60) - >>> self.eval() - >>> inputs = torch.rand(1, 3, 32, 32) - >>> level_outputs = self.forward(inputs) - >>> for level_out in level_outputs: - ... print(tuple(level_out.shape)) - (1, 128, 8, 8) - (1, 256, 4, 4) - (1, 512, 2, 2) - (1, 1024, 1, 1) - """ - - arch_settings = { - 34: (BasicBlock, [1, 1, 1, 2, 2, 1], [16, 32, 64, 128, 256, 512]), - 46: (Bottleneck, [1, 1, 1, 2, 2, 1], [16, 32, 64, 64, 128, 256]), - 60: (Bottleneck, [1, 1, 1, 2, 3, 1], [16, 32, 128, 256, 512, 1024]), - 102: (Bottleneck, [1, 1, 1, 3, 4, 1], [16, 32, 128, 256, 512, 1024]), - 169: (Bottleneck, [1, 1, 2, 3, 5, 1], [16, 32, 128, 256, 512, 1024]), - } - - make_stage_plugins = ResNet.make_stage_plugins - - def __init__( - self, - depth, - in_channels=3, - num_stages=4, - out_indices=(0, 1, 2, 3), - strides=(2, 2, 2, 2), - style="pytorch", - frozen_stages=-1, - conv_cfg=None, - norm_cfg=dict(type="BN", requires_grad=True), - norm_eval=True, - dcn=None, - stage_with_dcn=(False, False, False, False), - plugins=None, - with_cp=False, - zero_init_residual=True, - stage_with_level_root=(False, True, True, True), - residual_root=False, - ): - super(DLANetCustom, self).__init__() - if depth not in self.arch_settings: - raise KeyError(f"invalid depth {depth} for DLA") - block, levels, channels = self.arch_settings[depth] - self.conv_cfg = conv_cfg - self.norm_cfg = norm_cfg - self.zero_init_residual = zero_init_residual - self.frozen_stages = frozen_stages - self.num_stages = num_stages - assert num_stages >= 1 and num_stages <= 4 - self.out_indices = out_indices - assert max(out_indices) < num_stages - self.style = style - self.with_cp = with_cp - self.norm_eval = norm_eval - self.dcn = dcn - self.stage_with_dcn = stage_with_dcn - self.base_layer = nn.Sequential( - build_conv_layer( - self.conv_cfg, - in_channels, - channels[0], - kernel_size=7, - stride=1, - padding=3, - bias=False, - ), - build_norm_layer(self.norm_cfg, channels[0])[1], - nn.ReLU(inplace=True), - ) - - for i in range(2): - level_layer = self._make_conv_level( - channels[0], channels[i], levels[i], stride=i + 1 - ) - layer_name = f"level{i}" - self.add_module(layer_name, level_layer) - - for i in range(self.num_stages): - dcn = self.dcn if self.stage_with_dcn[i] else None - if plugins is not None: - stage_plugins = self.make_stage_plugins(plugins, i) - else: - stage_plugins = None - dla_layer = Tree( - levels[i + 2], - block, - channels[i + 1], - channels[i + 2], - strides[i], - level_root=stage_with_level_root[i], - root_residual=residual_root, - conv_cfg=self.conv_cfg, - norm_cfg=self.norm_cfg, - dcn=dcn, - plugins=stage_plugins, - style=self.style, - ) - layer_name = f"layer{i + 1}" - self.add_module(layer_name, dla_layer) - - self._freeze_stages() - - def _make_conv_level(self, inplanes, planes, convs, stride=1, dilation=1): - modules = [] - for i in range(convs): - modules.extend( - [ - build_conv_layer( - self.conv_cfg, - inplanes, - planes, - kernel_size=3, - stride=stride if i == 0 else 1, - padding=dilation, - bias=False, - dilation=dilation, - ), - build_norm_layer(self.norm_cfg, planes)[1], - nn.ReLU(inplace=True), - ] - ) - inplanes = planes - return nn.Sequential(*modules) - - def _freeze_stages(self): - if self.frozen_stages >= 0: - self.base_layer.eval() - for param in self.base_layer.parameters(): - param.requires_grad = False - - for i in range(2): - m = getattr(self, f"level{i}") - m.eval() - for param in m.parameters(): - param.requires_grad = False - - for i in range(1, self.frozen_stages + 1): - m = getattr(self, f"layer{i}") - m.eval() - for param in m.parameters(): - param.requires_grad = False - - def init_weights(self, pretrained=None): - """Initialize the weights in backbone. - - Args: - pretrained (str, optional): Path to pre-trained weights. - Defaults to None. - """ - if isinstance(pretrained, str): - logger = get_root_logger() - load_checkpoint(self, pretrained, strict=False, logger=logger) - elif pretrained is None: - for m in self.modules(): - if isinstance(m, nn.Conv2d): - n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels - m.weight.data.normal_(0, torch.tensor(2.0 / n).sqrt()) - elif isinstance(m, (_BatchNorm, nn.GroupNorm)): - m.weight.data.fill_(1) - m.bias.data.zero_() - - if self.dcn is not None: - for m in self.modules(): - if isinstance(m, Bottleneck) and hasattr( - m.conv2, "conv_offset" - ): - constant_init(m.conv2.conv_offset, 0) - - if self.zero_init_residual: - for m in self.modules(): - if isinstance(m, Bottleneck): - constant_init(m.norm3, 0) - elif isinstance(m, BasicBlock): - constant_init(m.norm2, 0) - else: - raise TypeError("pretrained must be a str or None") - - def forward(self, x): - """Forward function.""" - x = self.base_layer(x) - for i in range(2): - x = getattr(self, "level{}".format(i))(x) - outs = [] - for i, index in enumerate(range(1, self.num_stages + 1)): - x = getattr(self, "layer{}".format(index))(x) - if i in self.out_indices: - outs.append(x) - return tuple(outs) - - def train(self, mode=True): - """Convert the model into training mode while keep normalization layer - freezed.""" - super(DLANetCustom, self).train(mode) - self._freeze_stages() - if mode and self.norm_eval: - for m in self.modules(): - # trick: eval have effect on BatchNorm only - if isinstance(m, _BatchNorm): - m.eval() From e773170779908782332e35694e53feaa1d2b0dbe Mon Sep 17 00:00:00 2001 From: ArchieAlexArkhipov Date: Mon, 15 May 2023 13:59:00 +0000 Subject: [PATCH 13/25] upd formatter --- .../centernet_resnet18_dcnv2_140e_coco.py | 4 +- tools/analysis_tools/analyze_results.py | 55 +++++-------------- 2 files changed, 15 insertions(+), 44 deletions(-) diff --git a/configs/centernet/centernet_resnet18_dcnv2_140e_coco.py b/configs/centernet/centernet_resnet18_dcnv2_140e_coco.py index 67ae753c..898e1db3 100644 --- a/configs/centernet/centernet_resnet18_dcnv2_140e_coco.py +++ b/configs/centernet/centernet_resnet18_dcnv2_140e_coco.py @@ -34,9 +34,7 @@ ) # We fixed the incorrect img_norm_cfg problem in the source code. -img_norm_cfg = dict( - mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True -) +img_norm_cfg = dict(mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) train_pipeline = [ dict(type="LoadImageFromFile", to_float32=True, color_type="color"), diff --git a/tools/analysis_tools/analyze_results.py b/tools/analysis_tools/analyze_results.py index 1fafb771..dd106671 100644 --- a/tools/analysis_tools/analyze_results.py +++ b/tools/analysis_tools/analyze_results.py @@ -41,9 +41,7 @@ def bbox_map_eval(det_result, annotation, nproc=4): else: bbox_det_result = [det_result] # mAP - iou_thrs = np.linspace( - 0.5, 0.95, int(np.round((0.95 - 0.5) / 0.05)) + 1, endpoint=True - ) + iou_thrs = np.linspace(0.5, 0.95, int(np.round((0.95 - 0.5) / 0.05)) + 1, endpoint=True) processes = [] workers = Pool(processes=nproc) @@ -80,17 +78,13 @@ class ResultVisualizer: with the prediction result. Default: False. """ - def __init__( - self, show=False, wait_time=0, score_thr=0, overlay_gt_pred=False - ): + def __init__(self, show=False, wait_time=0, score_thr=0, overlay_gt_pred=False): self.show = show self.wait_time = wait_time self.score_thr = score_thr self.overlay_gt_pred = overlay_gt_pred - def _save_image_gts_results( - self, dataset, results, performances, out_dir=None - ): + def _save_image_gts_results(self, dataset, results, performances, out_dir=None): """Display or save image with groung truths and predictions from a model. @@ -155,18 +149,12 @@ def evaluate_and_show(self, dataset, results, topk=20, show_dir="work_dir"): topk = len(dataset) // 2 if isinstance(results[0], dict): - good_samples, bad_samples = self.panoptic_evaluate( - dataset, results, topk=topk - ) + good_samples, bad_samples = self.panoptic_evaluate(dataset, results, topk=topk) elif isinstance(results[0], list): - good_samples, bad_samples = self.detection_evaluate( - dataset, results, topk=topk - ) + good_samples, bad_samples = self.detection_evaluate(dataset, results, topk=topk) elif isinstance(results[0], tuple): results_ = [result[0] for result in results] - good_samples, bad_samples = self.detection_evaluate( - dataset, results_, topk=topk - ) + good_samples, bad_samples = self.detection_evaluate(dataset, results_, topk=topk) else: raise "The format of result is not supported yet. " "Current dict for panoptic segmentation and list " "or tuple for object detection are supported." @@ -263,9 +251,7 @@ def panoptic_evaluate(self, dataset, results, topk=20): dataset.file_client, print_log=False, ) - pq_results, classwise_results = pq_stat.pq_average( - dataset.categories, isthing=None - ) + pq_results, classwise_results = pq_stat.pq_average(dataset.categories, isthing=None) pqs[i] = pq_results["pq"] prog_bar.update() @@ -281,16 +267,10 @@ def panoptic_evaluate(self, dataset, results, topk=20): def parse_args(): - parser = argparse.ArgumentParser( - description="MMDet eval image prediction result for each" - ) + parser = argparse.ArgumentParser(description="MMDet eval image prediction result for each") parser.add_argument("config", help="test config file path") - parser.add_argument( - "prediction_path", help="prediction path where test pkl result" - ) - parser.add_argument( - "show_dir", help="directory where painted images will be saved" - ) + parser.add_argument("prediction_path", help="prediction path where test pkl result") + parser.add_argument("show_dir", help="directory where painted images will be saved") parser.add_argument("--show", action="store_true", help="show results") parser.add_argument( "--wait-time", @@ -302,8 +282,7 @@ def parse_args(): "--topk", default=20, type=int, - help="saved Number of the highest topk " - "and lowest topk after index sorting", + help="saved Number of the highest topk " "and lowest topk after index sorting", ) parser.add_argument( "--show-score-thr", @@ -359,21 +338,15 @@ def main(): "RepeatDataset", "ConcatDataset", ): - cfg.data.test.pipeline = get_loading_pipeline( - cfg.data.train.dataset.pipeline - ) + cfg.data.test.pipeline = get_loading_pipeline(cfg.data.train.dataset.pipeline) else: cfg.data.test.pipeline = get_loading_pipeline(cfg.data.train.pipeline) dataset = build_dataset(cfg.data.test) outputs = mmcv.load(args.prediction_path) - result_visualizer = ResultVisualizer( - args.show, args.wait_time, args.show_score_thr, args.overlay_gt_pred - ) - result_visualizer.evaluate_and_show( - dataset, outputs, topk=args.topk, show_dir=args.show_dir - ) + result_visualizer = ResultVisualizer(args.show, args.wait_time, args.show_score_thr, args.overlay_gt_pred) + result_visualizer.evaluate_and_show(dataset, outputs, topk=args.topk, show_dir=args.show_dir) if __name__ == "__main__": From ff77113382274bbb960a5a0ea64e3704d4c71706 Mon Sep 17 00:00:00 2001 From: ArchieAlexArkhipov Date: Thu, 18 May 2023 16:59:27 +0000 Subject: [PATCH 14/25] fix docs and name cycly centernet head --- mmdet/models/dense_heads/__init__.py | 4 +- .../dense_heads/cycle_centernet_head.py | 47 +++++++------------ 2 files changed, 18 insertions(+), 33 deletions(-) diff --git a/mmdet/models/dense_heads/__init__.py b/mmdet/models/dense_heads/__init__.py index 725cfec7..409b11a5 100644 --- a/mmdet/models/dense_heads/__init__.py +++ b/mmdet/models/dense_heads/__init__.py @@ -41,7 +41,7 @@ from .yolo_head import YOLOV3Head from .yolof_head import YOLOFHead from .yolox_head import YOLOXHead -from .cycle_centernet_head import CycleCenterNetHeadL1 +from .cycle_centernet_head import CycleCenterNetHead __all__ = [ "AnchorFreeHead", @@ -92,5 +92,5 @@ "Mask2FormerHead", "SOLOV2Head", "DDODHead", - "CycleCenterNetHeadL1", + "CycleCenterNetHead", ] diff --git a/mmdet/models/dense_heads/cycle_centernet_head.py b/mmdet/models/dense_heads/cycle_centernet_head.py index 3f016171..29c55420 100644 --- a/mmdet/models/dense_heads/cycle_centernet_head.py +++ b/mmdet/models/dense_heads/cycle_centernet_head.py @@ -1,7 +1,7 @@ # Copyright (c) OpenMMLab. All rights reserved. import torch import torch.nn as nn -from numpy import pi, exp +from numpy import pi from mmcv.cnn import bias_init_with_prob, normal_init from mmcv.ops import batched_nms from mmcv.runner import force_fp32 @@ -9,26 +9,20 @@ from mmdet.core import multi_apply from mmdet.models import HEADS, build_loss from mmdet.models.utils import gaussian_radius, gen_gaussian_target -from ..utils.gaussian_target import ( - get_local_maximum, - get_topk_from_heatmap, - transpose_and_gather_feat, -) +from ..utils.gaussian_target import get_local_maximum, get_topk_from_heatmap, transpose_and_gather_feat from .base_dense_head import BaseDenseHead from .dense_test_mixins import BBoxTestMixin @HEADS.register_module() -class CycleCenterNetHeadL1(BaseDenseHead, BBoxTestMixin): +class CycleCenterNetHead(BaseDenseHead, BBoxTestMixin): """Parsing Table Structures in the Wild Head. CycleCenterHead use - center_point to indicate object's position. + center point to indicate cell's position in a table. Paper link Args: in_channel (int): Number of channel in the input feature map. feat_channel (int): Number of channel in the intermediate feature map. - num_classes (int): Number of categories excluding the background - category. loss_center_heatmap (dict | None): Config of center heatmap loss. Default: GaussianFocalLoss. loss_wh (dict | None): Config of wh loss. Default: L1Loss. @@ -44,7 +38,6 @@ def __init__( self, in_channel, feat_channel, - num_classes, loss_center_heatmap=dict(type="GaussianFocalLoss", loss_weight=1.0), loss_offset=dict(type="L1Loss", loss_weight=1.0), loss_c2v=dict(type="L1Loss", loss_weight=1.0), @@ -53,9 +46,8 @@ def __init__( test_cfg=None, init_cfg=None, ): - super(CycleCenterNetHeadL1, self).__init__(init_cfg) - self.num_classes = num_classes - self.heatmap_head = self._build_head(in_channel, feat_channel, 2 * num_classes) + super(CycleCenterNetHead, self).__init__(init_cfg) + self.heatmap_head = self._build_head(in_channel, feat_channel, 2) self.offset_head = self._build_head(in_channel, feat_channel, 2) self.center2vertex_head = self._build_head(in_channel, feat_channel, 8) self.vertex2center_head = self._build_head(in_channel, feat_channel, 8) @@ -100,7 +92,7 @@ def forward(self, feats): Returns: center_heatmap_preds (List[Tensor]): center predict heatmaps for - all levels, the channels number is num_classes. + all levels, the channels number is 2. wh_preds (List[Tensor]): wh predicts for all levels, the channels number is 2. offset_preds (List[Tensor]): offset predicts for all levels, the @@ -116,7 +108,7 @@ def forward_single(self, feat): Returns: center_heatmap_pred (Tensor): center predict heatmaps, the - channels number is num_classes. + channels number is 2. wh_pred (Tensor): wh predicts, the channels number is 2. offset_pred (Tensor): offset predicts, the channels number is 2. """ @@ -148,15 +140,14 @@ def loss( gt_bboxes, gt_labels, img_metas, + # gt_masks=None, gt_bboxes_ignore=None, ): """Compute losses of the head. Args: center_heatmap_preds (list[Tensor]): center predict heatmaps for - all levels with shape (B, num_classes, H, W). - wh_preds (list[Tensor]): wh predicts for all levels with - shape (B, 2, H, W). + all levels with shape (B, 2, H, W). offset_preds (list[Tensor]): offset predicts for all levels with shape (B, 2, H, W). center2vertex_pred (list[Tensor]): center2vertex predicts for all levels @@ -168,8 +159,6 @@ def loss( gt_labels (list[Tensor]): class indices corresponding to each box. img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. - gt_bboxes_ignore (None | list[Tensor]): specify which bounding - boxes can be ignored when computing the loss. Default: None Returns: dict[str, Tensor]: which has components below: @@ -253,7 +242,7 @@ def get_targets(self, gt_bboxes, gt_labels, feat_shape, img_shape, c2v_pred, v2c tuple[dict,float]: The float value is mean avg_factor, the dict has components below: - center_heatmap_target (Tensor): targets of center heatmap, \ - shape (B, num_classes, H, W). + shape (B, 2, H, W). - wh_target (Tensor): targets of wh predict, shape \ (B, 2, H, W). - offset_target (Tensor): targets of offset predict, shape \ @@ -271,7 +260,7 @@ def get_targets(self, gt_bboxes, gt_labels, feat_shape, img_shape, c2v_pred, v2c width_ratio = float(feat_w / img_w) height_ratio = float(feat_h / img_h) - center_heatmap_target = gt_bboxes[-1].new_zeros([bs, 2 * self.num_classes, feat_h, feat_w]) + center_heatmap_target = gt_bboxes[-1].new_zeros([bs, 2, feat_h, feat_w]) offset_target = gt_bboxes[-1].new_zeros([bs, 2, feat_h, feat_w]) offset_target_weight = gt_bboxes[-1].new_zeros([bs, 2, feat_h, feat_w]) c2v_target = gt_bboxes[-1].new_zeros([bs, 8, feat_h, feat_w]) @@ -429,7 +418,7 @@ def get_bboxes( Args: center_heatmap_preds (list[Tensor]): Center predict heatmaps for - all levels with shape (B, num_classes, H, W). + all levels with shape (B, 2, H, W). wh_preds (list[Tensor]): WH predicts for all levels with shape (B, 2, H, W). offset_preds (list[Tensor]): Offset predicts for all levels @@ -513,15 +502,11 @@ def _get_bboxes_single( Args: center_heatmap_pred (Tensor): Center heatmap for current level with - shape (1, num_classes, H, W). + shape (1, 2, H, W). wh_pred (Tensor): WH heatmap for current level with shape - (1, num_classes, H, W). + (1, 2, H, W). offset_pred (Tensor): Offset for current level with shape (1, corner_offset_channels, H, W). - # center2vertex_pred (list[Tensor]): center2vertex predicts for all levels - # with shape (1, 8, H, W). - # vertex2center_pred (list[Tensor]): vertex2center predicts for all levels - # with shape (1, 8, H, W). img_meta (dict): Meta information of current image, e.g., image size, scaling factor, etc. rescale (bool): If True, return boxes in original image space. @@ -571,7 +556,7 @@ def decode_heatmap( Args: center_heatmap_pred (Tensor): center predict heatmap, - shape (B, num_classes, H, W). + shape (B, 2, H, W). wh_pred (Tensor): wh predict, shape (B, 2, H, W). offset_pred (Tensor): offset predict, shape (B, 2, H, W). img_shape (list[int]): image shape in [h, w] format. From b870fa2dc5c5d38e78110a080ee8148035db98ff Mon Sep 17 00:00:00 2001 From: ArchieAlexArkhipov Date: Sun, 21 May 2023 20:27:41 +0000 Subject: [PATCH 15/25] constant radius --- .../dense_heads/cycle_centernet_head.py | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/mmdet/models/dense_heads/cycle_centernet_head.py b/mmdet/models/dense_heads/cycle_centernet_head.py index 29c55420..533fb477 100644 --- a/mmdet/models/dense_heads/cycle_centernet_head.py +++ b/mmdet/models/dense_heads/cycle_centernet_head.py @@ -267,12 +267,15 @@ def get_targets(self, gt_bboxes, gt_labels, feat_shape, img_shape, c2v_pred, v2c v2c_target = gt_bboxes[-1].new_zeros([bs, 8, feat_h, feat_w]) pairing_weight = gt_bboxes[-1].new_zeros([bs, 8, feat_h, feat_w]) + radius = gaussian_radius([feat_h / 8, feat_w / 8], min_overlap=0.3) + radius = max(0, int(radius)) + for batch_id in range(bs): gt_bbox = gt_bboxes[batch_id] center_x = (gt_bbox[:, [0]] + gt_bbox[:, [2]]) * width_ratio / 2 center_y = (gt_bbox[:, [1]] + gt_bbox[:, [3]]) * height_ratio / 2 gt_centers = torch.cat((center_x, center_y), dim=1) - vertexes = {} + vertexes = set() for j, ct in enumerate(gt_centers): ctx_int, cty_int = ct.int() ctx, cty = ct @@ -280,12 +283,10 @@ def get_targets(self, gt_bboxes, gt_labels, feat_shape, img_shape, c2v_pred, v2c scale_box_w = (gt_bbox[j][2] - gt_bbox[j][0]) * width_ratio scale_box_h_r = scale_box_h / 2 scale_box_w_r = scale_box_w / 2 - radius = gaussian_radius([scale_box_h, scale_box_w], min_overlap=0.3) - radius = max(0, int(radius)) gen_gaussian_target( - center_heatmap_target[batch_id, 0], - [ctx_int, cty_int], - radius, + heatmap=center_heatmap_target[batch_id, 0], + center=[ctx_int, cty_int], + radius=radius, ) tl_x, tr_x, br_x, bl_x = map( @@ -305,17 +306,13 @@ def get_targets(self, gt_bboxes, gt_labels, feat_shape, img_shape, c2v_pred, v2c lambda x: feat_h - 1 if x >= feat_h else x if x >= 0 else 0, (tl_y_int, tr_y_int, br_y_int, bl_y_int), ) - vtx_gaus_r = max(0, int(gaussian_radius([scale_box_h_r, scale_box_w_r], min_overlap=0.3))) for x, y in ( (tl_x_int, tl_y_int), (tr_x_int, tr_y_int), (br_x_int, br_y_int), (bl_x_int, bl_y_int), ): - if (x, y) in vertexes: - vertexes[(x, y)].append(vtx_gaus_r) - else: - vertexes[(x, y)] = [vtx_gaus_r] + vertexes.add((x, y)) offset_target[batch_id, 0, cty_int, ctx_int] = ctx - ctx_int offset_target[batch_id, 0, tl_y_int, tl_x_int] = tl_x - tl_x_int @@ -378,11 +375,11 @@ def get_targets(self, gt_bboxes, gt_labels, feat_shape, img_shape, c2v_pred, v2c w = 1 - torch.exp(-pi * D_cv) pairing_weight[batch_id, 2 * idx + k, cty_int, ctx_int] += w pairing_weight[batch_id, 2 * idx + k, v_y_int, v_x_int] += w - for point, radiuses in vertexes.items(): + for x, y in vertexes: gen_gaussian_target( heatmap=center_heatmap_target[batch_id, 1], - center=list(point), - radius=max(0, int(sum(radiuses) / len(radiuses))), + center=[x, y], + radius=radius, ) avg_factor = max(1, center_heatmap_target.eq(1).sum()) From 1e382b7b3b15a7b666904183113b640ed4a8dfb1 Mon Sep 17 00:00:00 2001 From: ArchieAlexArkhipov Date: Sun, 21 May 2023 23:42:15 +0000 Subject: [PATCH 16/25] del set for vertexes --- mmdet/models/dense_heads/cycle_centernet_head.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/mmdet/models/dense_heads/cycle_centernet_head.py b/mmdet/models/dense_heads/cycle_centernet_head.py index 533fb477..ceb56e71 100644 --- a/mmdet/models/dense_heads/cycle_centernet_head.py +++ b/mmdet/models/dense_heads/cycle_centernet_head.py @@ -275,7 +275,6 @@ def get_targets(self, gt_bboxes, gt_labels, feat_shape, img_shape, c2v_pred, v2c center_x = (gt_bbox[:, [0]] + gt_bbox[:, [2]]) * width_ratio / 2 center_y = (gt_bbox[:, [1]] + gt_bbox[:, [3]]) * height_ratio / 2 gt_centers = torch.cat((center_x, center_y), dim=1) - vertexes = set() for j, ct in enumerate(gt_centers): ctx_int, cty_int = ct.int() ctx, cty = ct @@ -312,7 +311,12 @@ def get_targets(self, gt_bboxes, gt_labels, feat_shape, img_shape, c2v_pred, v2c (br_x_int, br_y_int), (bl_x_int, bl_y_int), ): - vertexes.add((x, y)) + if center_heatmap_target[batch_id, 1, y, x] != 1.0: + gen_gaussian_target( + heatmap=center_heatmap_target[batch_id, 1], + center=[x, y], + radius=radius, + ) offset_target[batch_id, 0, cty_int, ctx_int] = ctx - ctx_int offset_target[batch_id, 0, tl_y_int, tl_x_int] = tl_x - tl_x_int @@ -375,12 +379,6 @@ def get_targets(self, gt_bboxes, gt_labels, feat_shape, img_shape, c2v_pred, v2c w = 1 - torch.exp(-pi * D_cv) pairing_weight[batch_id, 2 * idx + k, cty_int, ctx_int] += w pairing_weight[batch_id, 2 * idx + k, v_y_int, v_x_int] += w - for x, y in vertexes: - gen_gaussian_target( - heatmap=center_heatmap_target[batch_id, 1], - center=[x, y], - radius=radius, - ) avg_factor = max(1, center_heatmap_target.eq(1).sum()) target_result = dict( From 8589e2548ca36694671502d080b8c6905e3ff998 Mon Sep 17 00:00:00 2001 From: ArchieAlexArkhipov Date: Mon, 22 May 2023 03:49:19 +0000 Subject: [PATCH 17/25] quad via mask --- mmdet/models/dense_heads/base_dense_head.py | 258 +++++++++--------- .../dense_heads/cycle_centernet_head.py | 63 ++--- mmdet/models/detectors/single_stage.py | 43 +-- 3 files changed, 171 insertions(+), 193 deletions(-) diff --git a/mmdet/models/dense_heads/base_dense_head.py b/mmdet/models/dense_heads/base_dense_head.py index 0c7abb7b..31726870 100644 --- a/mmdet/models/dense_heads/base_dense_head.py +++ b/mmdet/models/dense_heads/base_dense_head.py @@ -20,7 +20,7 @@ def init_weights(self): # avoid init_cfg overwrite the initialization of `conv_offset` for m in self.modules(): # DeformConv2dPack, ModulatedDeformConv2dPack - if hasattr(m, 'conv_offset'): + if hasattr(m, "conv_offset"): constant_init(m.conv_offset, 0) @abstractmethod @@ -28,16 +28,18 @@ def loss(self, **kwargs): """Compute losses of the head.""" pass - @force_fp32(apply_to=('cls_scores', 'bbox_preds')) - def get_bboxes(self, - cls_scores, - bbox_preds, - score_factors=None, - img_metas=None, - cfg=None, - rescale=False, - with_nms=True, - **kwargs): + @force_fp32(apply_to=("cls_scores", "bbox_preds")) + def get_bboxes( + self, + cls_scores, + bbox_preds, + score_factors=None, + img_metas=None, + cfg=None, + rescale=False, + with_nms=True, + **kwargs + ): """Transform network outputs of a batch into bbox results. Note: When score_factors is not None, the cls_scores are @@ -84,9 +86,8 @@ def get_bboxes(self, featmap_sizes = [cls_scores[i].shape[-2:] for i in range(num_levels)] mlvl_priors = self.prior_generator.grid_priors( - featmap_sizes, - dtype=cls_scores[0].dtype, - device=cls_scores[0].device) + featmap_sizes, dtype=cls_scores[0].dtype, device=cls_scores[0].device + ) result_list = [] @@ -99,23 +100,32 @@ def get_bboxes(self, else: score_factor_list = [None for _ in range(num_levels)] - results = self._get_bboxes_single(cls_score_list, bbox_pred_list, - score_factor_list, mlvl_priors, - img_meta, cfg, rescale, with_nms, - **kwargs) + results = self._get_bboxes_single( + cls_score_list, + bbox_pred_list, + score_factor_list, + mlvl_priors, + img_meta, + cfg, + rescale, + with_nms, + **kwargs + ) result_list.append(results) return result_list - def _get_bboxes_single(self, - cls_score_list, - bbox_pred_list, - score_factor_list, - mlvl_priors, - img_meta, - cfg, - rescale=False, - with_nms=True, - **kwargs): + def _get_bboxes_single( + self, + cls_score_list, + bbox_pred_list, + score_factor_list, + mlvl_priors, + img_meta, + cfg, + rescale=False, + with_nms=True, + **kwargs + ): """Transform outputs of a single image into bbox predictions. Args: @@ -164,8 +174,8 @@ def _get_bboxes_single(self, with_score_factors = True cfg = self.test_cfg if cfg is None else cfg - img_shape = img_meta['img_shape'] - nms_pre = cfg.get('nms_pre', -1) + img_shape = img_meta["img_shape"] + nms_pre = cfg.get("nms_pre", -1) mlvl_bboxes = [] mlvl_scores = [] @@ -174,18 +184,15 @@ def _get_bboxes_single(self, mlvl_score_factors = [] else: mlvl_score_factors = None - for level_idx, (cls_score, bbox_pred, score_factor, priors) in \ - enumerate(zip(cls_score_list, bbox_pred_list, - score_factor_list, mlvl_priors)): - + for level_idx, (cls_score, bbox_pred, score_factor, priors) in enumerate( + zip(cls_score_list, bbox_pred_list, score_factor_list, mlvl_priors) + ): assert cls_score.size()[-2:] == bbox_pred.size()[-2:] bbox_pred = bbox_pred.permute(1, 2, 0).reshape(-1, 4) if with_score_factors: - score_factor = score_factor.permute(1, 2, - 0).reshape(-1).sigmoid() - cls_score = cls_score.permute(1, 2, - 0).reshape(-1, self.cls_out_channels) + score_factor = score_factor.permute(1, 2, 0).reshape(-1).sigmoid() + cls_score = cls_score.permute(1, 2, 0).reshape(-1, self.cls_out_channels) if self.use_sigmoid_cls: scores = cls_score.sigmoid() else: @@ -200,18 +207,17 @@ def _get_bboxes_single(self, # find a slight drop in performance, you can set a larger # `nms_pre` than before. results = filter_scores_and_topk( - scores, cfg.score_thr, nms_pre, - dict(bbox_pred=bbox_pred, priors=priors)) + scores, cfg.score_thr, nms_pre, dict(bbox_pred=bbox_pred, priors=priors) + ) scores, labels, keep_idxs, filtered_results = results - bbox_pred = filtered_results['bbox_pred'] - priors = filtered_results['priors'] + bbox_pred = filtered_results["bbox_pred"] + priors = filtered_results["priors"] if with_score_factors: score_factor = score_factor[keep_idxs] - bboxes = self.bbox_coder.decode( - priors, bbox_pred, max_shape=img_shape) + bboxes = self.bbox_coder.decode(priors, bbox_pred, max_shape=img_shape) mlvl_bboxes.append(bboxes) mlvl_scores.append(scores) @@ -219,20 +225,30 @@ def _get_bboxes_single(self, if with_score_factors: mlvl_score_factors.append(score_factor) - return self._bbox_post_process(mlvl_scores, mlvl_labels, mlvl_bboxes, - img_meta['scale_factor'], cfg, rescale, - with_nms, mlvl_score_factors, **kwargs) - - def _bbox_post_process(self, - mlvl_scores, - mlvl_labels, - mlvl_bboxes, - scale_factor, - cfg, - rescale=False, - with_nms=True, - mlvl_score_factors=None, - **kwargs): + return self._bbox_post_process( + mlvl_scores, + mlvl_labels, + mlvl_bboxes, + img_meta["scale_factor"], + cfg, + rescale, + with_nms, + mlvl_score_factors, + **kwargs + ) + + def _bbox_post_process( + self, + mlvl_scores, + mlvl_labels, + mlvl_bboxes, + scale_factor, + cfg, + rescale=False, + with_nms=True, + mlvl_score_factors=None, + **kwargs + ): """bbox post-processing method. The boxes would be rescaled to the original image scale and do @@ -292,22 +308,24 @@ def _bbox_post_process(self, det_bboxes = torch.cat([mlvl_bboxes, mlvl_scores[:, None]], -1) return det_bboxes, mlvl_labels - det_bboxes, keep_idxs = batched_nms(mlvl_bboxes, mlvl_scores, - mlvl_labels, cfg.nms) - det_bboxes = det_bboxes[:cfg.max_per_img] - det_labels = mlvl_labels[keep_idxs][:cfg.max_per_img] + det_bboxes, keep_idxs = batched_nms(mlvl_bboxes, mlvl_scores, mlvl_labels, cfg.nms) + det_bboxes = det_bboxes[: cfg.max_per_img] + det_labels = mlvl_labels[keep_idxs][: cfg.max_per_img] return det_bboxes, det_labels else: return mlvl_bboxes, mlvl_scores, mlvl_labels - def forward_train(self, - x, - img_metas, - gt_bboxes, - gt_labels=None, - gt_bboxes_ignore=None, - proposal_cfg=None, - **kwargs): + def forward_train( + self, + x, + img_metas, + gt_bboxes, + gt_labels=None, + gt_masks=None, + gt_bboxes_ignore=None, + proposal_cfg=None, + **kwargs + ): """ Args: x (list[Tensor]): Features from FPN. @@ -332,12 +350,11 @@ def forward_train(self, loss_inputs = outs + (gt_bboxes, img_metas) else: loss_inputs = outs + (gt_bboxes, gt_labels, img_metas) - losses = self.loss(*loss_inputs, gt_bboxes_ignore=gt_bboxes_ignore) + losses = self.loss(*loss_inputs, gt_masks=gt_masks, gt_bboxes_ignore=gt_bboxes_ignore) if proposal_cfg is None: return losses else: - proposal_list = self.get_bboxes( - *outs, img_metas=img_metas, cfg=proposal_cfg) + proposal_list = self.get_bboxes(*outs, img_metas=img_metas, cfg=proposal_cfg) return losses, proposal_list def simple_test(self, feats, img_metas, rescale=False): @@ -359,13 +376,8 @@ def simple_test(self, feats, img_metas, rescale=False): """ return self.simple_test_bboxes(feats, img_metas, rescale=rescale) - @force_fp32(apply_to=('cls_scores', 'bbox_preds')) - def onnx_export(self, - cls_scores, - bbox_preds, - score_factors=None, - img_metas=None, - with_nms=True): + @force_fp32(apply_to=("cls_scores", "bbox_preds")) + def onnx_export(self, cls_scores, bbox_preds, score_factors=None, img_metas=None, with_nms=True): """Transform network output for a batch into bbox predictions. Args: @@ -395,25 +407,21 @@ def onnx_export(self, featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] mlvl_priors = self.prior_generator.grid_priors( - featmap_sizes, - dtype=bbox_preds[0].dtype, - device=bbox_preds[0].device) + featmap_sizes, dtype=bbox_preds[0].dtype, device=bbox_preds[0].device + ) mlvl_cls_scores = [cls_scores[i].detach() for i in range(num_levels)] mlvl_bbox_preds = [bbox_preds[i].detach() for i in range(num_levels)] - assert len( - img_metas - ) == 1, 'Only support one input image while in exporting to ONNX' - img_shape = img_metas[0]['img_shape_for_onnx'] + assert len(img_metas) == 1, "Only support one input image while in exporting to ONNX" + img_shape = img_metas[0]["img_shape_for_onnx"] cfg = self.test_cfg assert len(cls_scores) == len(bbox_preds) == len(mlvl_priors) device = cls_scores[0].device batch_size = cls_scores[0].shape[0] # convert to tensor to keep tracing - nms_pre_tensor = torch.tensor( - cfg.get('nms_pre', -1), device=device, dtype=torch.long) + nms_pre_tensor = torch.tensor(cfg.get("nms_pre", -1), device=device, dtype=torch.long) # e.g. Retina, FreeAnchor, etc. if score_factors is None: @@ -422,22 +430,18 @@ def onnx_export(self, else: # e.g. FCOS, PAA, ATSS, etc. with_score_factors = True - mlvl_score_factor = [ - score_factors[i].detach() for i in range(num_levels) - ] + mlvl_score_factor = [score_factors[i].detach() for i in range(num_levels)] mlvl_score_factors = [] mlvl_batch_bboxes = [] mlvl_scores = [] for cls_score, bbox_pred, score_factors, priors in zip( - mlvl_cls_scores, mlvl_bbox_preds, mlvl_score_factor, - mlvl_priors): + mlvl_cls_scores, mlvl_bbox_preds, mlvl_score_factor, mlvl_priors + ): assert cls_score.size()[-2:] == bbox_pred.size()[-2:] - scores = cls_score.permute(0, 2, 3, - 1).reshape(batch_size, -1, - self.cls_out_channels) + scores = cls_score.permute(0, 2, 3, 1).reshape(batch_size, -1, self.cls_out_channels) if self.use_sigmoid_cls: scores = scores.sigmoid() nms_pre_score = scores @@ -446,18 +450,16 @@ def onnx_export(self, nms_pre_score = scores if with_score_factors: - score_factors = score_factors.permute(0, 2, 3, 1).reshape( - batch_size, -1).sigmoid() - bbox_pred = bbox_pred.permute(0, 2, 3, - 1).reshape(batch_size, -1, 4) + score_factors = score_factors.permute(0, 2, 3, 1).reshape(batch_size, -1).sigmoid() + bbox_pred = bbox_pred.permute(0, 2, 3, 1).reshape(batch_size, -1, 4) priors = priors.expand(batch_size, -1, priors.size(-1)) # Get top-k predictions from mmdet.core.export import get_k_for_topk + nms_pre = get_k_for_topk(nms_pre_tensor, bbox_pred.shape[1]) if nms_pre > 0: - if with_score_factors: - nms_pre_score = (nms_pre_score * score_factors[..., None]) + nms_pre_score = nms_pre_score * score_factors[..., None] else: nms_pre_score = nms_pre_score @@ -471,26 +473,22 @@ def onnx_export(self, max_scores, _ = nms_pre_score[..., :-1].max(-1) _, topk_inds = max_scores.topk(nms_pre) - batch_inds = torch.arange( - batch_size, device=bbox_pred.device).view( - -1, 1).expand_as(topk_inds).long() + batch_inds = ( + torch.arange(batch_size, device=bbox_pred.device).view(-1, 1).expand_as(topk_inds).long() + ) # Avoid onnx2tensorrt issue in https://github.com/NVIDIA/TensorRT/issues/1134 # noqa: E501 transformed_inds = bbox_pred.shape[1] * batch_inds + topk_inds - priors = priors.reshape( - -1, priors.size(-1))[transformed_inds, :].reshape( - batch_size, -1, priors.size(-1)) - bbox_pred = bbox_pred.reshape(-1, - 4)[transformed_inds, :].reshape( - batch_size, -1, 4) - scores = scores.reshape( - -1, self.cls_out_channels)[transformed_inds, :].reshape( - batch_size, -1, self.cls_out_channels) + priors = priors.reshape(-1, priors.size(-1))[transformed_inds, :].reshape( + batch_size, -1, priors.size(-1) + ) + bbox_pred = bbox_pred.reshape(-1, 4)[transformed_inds, :].reshape(batch_size, -1, 4) + scores = scores.reshape(-1, self.cls_out_channels)[transformed_inds, :].reshape( + batch_size, -1, self.cls_out_channels + ) if with_score_factors: - score_factors = score_factors.reshape( - -1, 1)[transformed_inds].reshape(batch_size, -1) + score_factors = score_factors.reshape(-1, 1)[transformed_inds].reshape(batch_size, -1) - bboxes = self.bbox_coder.decode( - priors, bbox_pred, max_shape=img_shape) + bboxes = self.bbox_coder.decode(priors, bbox_pred, max_shape=img_shape) mlvl_batch_bboxes.append(bboxes) mlvl_scores.append(scores) @@ -507,20 +505,24 @@ def onnx_export(self, from mmdet.core.export import add_dummy_nms_for_onnx if not self.use_sigmoid_cls: - batch_scores = batch_scores[..., :self.num_classes] + batch_scores = batch_scores[..., : self.num_classes] if with_score_factors: batch_scores = batch_scores * (batch_score_factors.unsqueeze(2)) if with_nms: - max_output_boxes_per_class = cfg.nms.get( - 'max_output_boxes_per_class', 200) - iou_threshold = cfg.nms.get('iou_threshold', 0.5) + max_output_boxes_per_class = cfg.nms.get("max_output_boxes_per_class", 200) + iou_threshold = cfg.nms.get("iou_threshold", 0.5) score_threshold = cfg.score_thr - nms_pre = cfg.get('deploy_nms_pre', -1) - return add_dummy_nms_for_onnx(batch_bboxes, batch_scores, - max_output_boxes_per_class, - iou_threshold, score_threshold, - nms_pre, cfg.max_per_img) + nms_pre = cfg.get("deploy_nms_pre", -1) + return add_dummy_nms_for_onnx( + batch_bboxes, + batch_scores, + max_output_boxes_per_class, + iou_threshold, + score_threshold, + nms_pre, + cfg.max_per_img, + ) else: return batch_bboxes, batch_scores diff --git a/mmdet/models/dense_heads/cycle_centernet_head.py b/mmdet/models/dense_heads/cycle_centernet_head.py index ceb56e71..463144ed 100644 --- a/mmdet/models/dense_heads/cycle_centernet_head.py +++ b/mmdet/models/dense_heads/cycle_centernet_head.py @@ -140,7 +140,7 @@ def loss( gt_bboxes, gt_labels, img_metas, - # gt_masks=None, + gt_masks=None, gt_bboxes_ignore=None, ): """Compute losses of the head. @@ -181,6 +181,7 @@ def loss( target_result, avg_factor = self.get_targets( gt_bboxes, gt_labels, + gt_masks, center_heatmap_pred.shape, img_metas[0]["pad_shape"], center2vertex_pred, @@ -225,7 +226,7 @@ def loss( loss_v2c=loss_v2c, ) - def get_targets(self, gt_bboxes, gt_labels, feat_shape, img_shape, c2v_pred, v2c_pred): + def get_targets(self, gt_bboxes, gt_labels, gt_masks, feat_shape, img_shape, c2v_pred, v2c_pred): """Compute regression and classification targets in multiple images and compute weight for Pairing Loss. @@ -271,29 +272,19 @@ def get_targets(self, gt_bboxes, gt_labels, feat_shape, img_shape, c2v_pred, v2c radius = max(0, int(radius)) for batch_id in range(bs): - gt_bbox = gt_bboxes[batch_id] - center_x = (gt_bbox[:, [0]] + gt_bbox[:, [2]]) * width_ratio / 2 - center_y = (gt_bbox[:, [1]] + gt_bbox[:, [3]]) * height_ratio / 2 - gt_centers = torch.cat((center_x, center_y), dim=1) - for j, ct in enumerate(gt_centers): - ctx_int, cty_int = ct.int() - ctx, cty = ct - scale_box_h = (gt_bbox[j][3] - gt_bbox[j][1]) * height_ratio - scale_box_w = (gt_bbox[j][2] - gt_bbox[j][0]) * width_ratio - scale_box_h_r = scale_box_h / 2 - scale_box_w_r = scale_box_w / 2 + for mask in gt_masks[batch_id].masks: + mask = mask[0] + ctx, cty = (mask[0] + mask[2] + mask[4] + mask[6]) / 4, (mask[1] + mask[3] + mask[5] + mask[7]) / 4 + ctx_int, cty_int = int(ctx * width_ratio), int(cty * height_ratio) + gen_gaussian_target( heatmap=center_heatmap_target[batch_id, 0], center=[ctx_int, cty_int], radius=radius, ) - tl_x, tr_x, br_x, bl_x = map( - lambda x: x * width_ratio, (gt_bbox[j][0], gt_bbox[j][2], gt_bbox[j][2], gt_bbox[j][0]) - ) - tl_y, tr_y, br_y, bl_y = map( - lambda x: x * height_ratio, (gt_bbox[j][1], gt_bbox[j][1], gt_bbox[j][3], gt_bbox[j][3]) - ) + tl_x, tr_x, br_x, bl_x = map(lambda x: x * width_ratio, (mask[0], mask[2], mask[4], mask[6])) + tl_y, tr_y, br_y, bl_y = map(lambda x: x * height_ratio, (mask[1], mask[3], mask[5], mask[7])) tl_x_int, tl_y_int, tr_x_int, tr_y_int, br_x_int, br_y_int, bl_x_int, bl_y_int = map( int, (tl_x, tl_y, tr_x, tr_y, br_x, br_y, bl_x, bl_y) ) @@ -336,23 +327,23 @@ def get_targets(self, gt_bboxes, gt_labels, feat_shape, img_shape, c2v_pred, v2c offset_target_weight[batch_id, :, br_y_int, br_x_int] = 1 offset_target_weight[batch_id, :, bl_y_int, bl_x_int] = 1 - c2v_target[batch_id, 0, cty_int, ctx_int] = -scale_box_w_r - c2v_target[batch_id, 1, cty_int, ctx_int] = -scale_box_h_r - c2v_target[batch_id, 2, cty_int, ctx_int] = scale_box_w_r - c2v_target[batch_id, 3, cty_int, ctx_int] = -scale_box_h_r - c2v_target[batch_id, 4, cty_int, ctx_int] = scale_box_w_r - c2v_target[batch_id, 5, cty_int, ctx_int] = scale_box_h_r - c2v_target[batch_id, 6, cty_int, ctx_int] = -scale_box_w_r - c2v_target[batch_id, 7, cty_int, ctx_int] = scale_box_h_r - - v2c_target[batch_id, 0, tl_y_int, tl_x_int] = scale_box_w_r - v2c_target[batch_id, 1, tl_y_int, tl_x_int] = scale_box_h_r - v2c_target[batch_id, 2, tr_y_int, tr_x_int] = -scale_box_w_r - v2c_target[batch_id, 3, tr_y_int, tr_x_int] = scale_box_h_r - v2c_target[batch_id, 4, br_y_int, br_x_int] = -scale_box_w_r - v2c_target[batch_id, 5, br_y_int, br_x_int] = -scale_box_h_r - v2c_target[batch_id, 6, bl_y_int, bl_x_int] = scale_box_w_r - v2c_target[batch_id, 7, bl_y_int, bl_x_int] = -scale_box_h_r + c2v_target[batch_id, 0, cty_int, ctx_int] = tl_x - ctx + c2v_target[batch_id, 1, cty_int, ctx_int] = tl_y - cty + c2v_target[batch_id, 2, cty_int, ctx_int] = tr_x - ctx + c2v_target[batch_id, 3, cty_int, ctx_int] = tr_y - cty + c2v_target[batch_id, 4, cty_int, ctx_int] = br_x - ctx + c2v_target[batch_id, 5, cty_int, ctx_int] = br_y - cty + c2v_target[batch_id, 6, cty_int, ctx_int] = bl_x - ctx + c2v_target[batch_id, 7, cty_int, ctx_int] = bl_y - cty + + v2c_target[batch_id, 0, tl_y_int, tl_x_int] = ctx - tl_x + v2c_target[batch_id, 1, tl_y_int, tl_x_int] = cty - tl_y + v2c_target[batch_id, 2, tr_y_int, tr_x_int] = ctx - tr_x + v2c_target[batch_id, 3, tr_y_int, tr_x_int] = cty - tr_y + v2c_target[batch_id, 4, br_y_int, br_x_int] = ctx - br_x + v2c_target[batch_id, 5, br_y_int, br_x_int] = cty - br_y + v2c_target[batch_id, 6, bl_y_int, bl_x_int] = ctx - bl_x + v2c_target[batch_id, 7, bl_y_int, bl_x_int] = cty - bl_y # Pairing loss for idx, v_x_int, v_y_int in ( diff --git a/mmdet/models/detectors/single_stage.py b/mmdet/models/detectors/single_stage.py index c375c72d..aeeef0d0 100644 --- a/mmdet/models/detectors/single_stage.py +++ b/mmdet/models/detectors/single_stage.py @@ -16,18 +16,12 @@ class SingleStageDetector(BaseDetector): output features of the backbone+neck. """ - def __init__(self, - backbone, - neck=None, - bbox_head=None, - train_cfg=None, - test_cfg=None, - pretrained=None, - init_cfg=None): + def __init__( + self, backbone, neck=None, bbox_head=None, train_cfg=None, test_cfg=None, pretrained=None, init_cfg=None + ): super(SingleStageDetector, self).__init__(init_cfg) if pretrained: - warnings.warn('DeprecationWarning: pretrained is deprecated, ' - 'please use "init_cfg" instead') + warnings.warn("DeprecationWarning: pretrained is deprecated, " 'please use "init_cfg" instead') backbone.pretrained = pretrained self.backbone = build_backbone(backbone) if neck is not None: @@ -54,12 +48,7 @@ def forward_dummy(self, img): outs = self.bbox_head(x) return outs - def forward_train(self, - img, - img_metas, - gt_bboxes, - gt_labels, - gt_bboxes_ignore=None): + def forward_train(self, img, img_metas, gt_bboxes, gt_labels, gt_masks=None, gt_bboxes_ignore=None): """ Args: img (Tensor): Input images of shape (N, C, H, W). @@ -80,8 +69,7 @@ def forward_train(self, """ super(SingleStageDetector, self).forward_train(img, img_metas) x = self.extract_feat(img) - losses = self.bbox_head.forward_train(x, img_metas, gt_bboxes, - gt_labels, gt_bboxes_ignore) + losses = self.bbox_head.forward_train(x, img_metas, gt_bboxes, gt_labels, gt_masks, gt_bboxes_ignore) return losses def simple_test(self, img, img_metas, rescale=False): @@ -99,8 +87,7 @@ def simple_test(self, img, img_metas, rescale=False): corresponds to each class. """ feat = self.extract_feat(img) - results_list = self.bbox_head.simple_test( - feat, img_metas, rescale=rescale) + results_list = self.bbox_head.simple_test(feat, img_metas, rescale=rescale) bbox_results = [ bbox2result(det_bboxes, det_labels, self.bbox_head.num_classes) for det_bboxes, det_labels in results_list @@ -125,13 +112,12 @@ def aug_test(self, imgs, img_metas, rescale=False): The outer list corresponds to each image. The inner list corresponds to each class. """ - assert hasattr(self.bbox_head, 'aug_test'), \ - f'{self.bbox_head.__class__.__name__}' \ - ' does not support test-time augmentation' + assert hasattr(self.bbox_head, "aug_test"), ( + f"{self.bbox_head.__class__.__name__}" " does not support test-time augmentation" + ) feats = self.extract_feats(imgs) - results_list = self.bbox_head.aug_test( - feats, img_metas, rescale=rescale) + results_list = self.bbox_head.aug_test(feats, img_metas, rescale=rescale) bbox_results = [ bbox2result(det_bboxes, det_labels, self.bbox_head.num_classes) for det_bboxes, det_labels in results_list @@ -155,17 +141,16 @@ def onnx_export(self, img, img_metas, with_nms=True): # get shape as tensor img_shape = torch._shape_as_tensor(img)[2:] - img_metas[0]['img_shape_for_onnx'] = img_shape + img_metas[0]["img_shape_for_onnx"] = img_shape # get pad input shape to support onnx dynamic shape for exporting # `CornerNet` and `CentripetalNet`, which 'pad_shape' is used # for inference - img_metas[0]['pad_shape_for_onnx'] = img_shape + img_metas[0]["pad_shape_for_onnx"] = img_shape if len(outs) == 2: # add dummy score_factor outs = (*outs, None) # TODO Can we change to `get_bboxes` when `onnx_export` fail - det_bboxes, det_labels = self.bbox_head.onnx_export( - *outs, img_metas, with_nms=with_nms) + det_bboxes, det_labels = self.bbox_head.onnx_export(*outs, img_metas, with_nms=with_nms) return det_bboxes, det_labels From 6daa0aefb5315500d0f96bbb63c3016315014f47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=85=D0=B8=D0=BF=D0=BE=D0=B2=20=D0=90=D0=BB?= =?UTF-8?q?=D0=B5=D0=BA=D1=81=D0=B0=D0=BD=D0=B4=D1=80?= <32227207+ArchieAlexArkhipov@users.noreply.github.com> Date: Mon, 26 Jun 2023 17:57:22 +0300 Subject: [PATCH 18/25] schema --- demo/schema.jpg | Bin 0 -> 97414 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 demo/schema.jpg diff --git a/demo/schema.jpg b/demo/schema.jpg new file mode 100644 index 0000000000000000000000000000000000000000..833313a0bb57b725d49c6df23fca7628d467f057 GIT binary patch literal 97414 zcmc%wbx@nZ_dkjUTA+oZrBJ+hDORAkwP^9;9*VnrAW%VyJHeq)ph$2F5=xN(0fKv= zXmE;qd(-#(`~A%KH}kuH+?hLbcX#$#$+LU*wYz8coRjD8?B8DiGF2rNB>)Z%0Dyz_ zfWHd>c>n<(9zNcE0(^Y@2LuEUh$x7N2nmU(NXdvPXsPIEX{czP(m#97LeI#-_>_kA zB^$>JZeBiKIu=1;0UqJ!JiI*r8o?oW@POzc(Gwz~Cp-)^3_Smj-QS-8@&`C`cz}C2 zEC5_`oO|Rre|rFQSR!%n{Ui5(+dR?Nn;U zK2n8+DASBa;zz^uFf{i6&=;ST`dUiblA$7UmVaz|>zLjoO)f%#Ge%UVZT z@Ris}49?I}%cS2p@;KWW@p;Fjsyh}9=Ul|4n(bw#QmbnmQwfVcg`G^c7dZ{*ax}TS z1>H>?nz=h2v6Ey12t>!dBm0k#A*D?zoE~fDu`OmNg@cr8@T^0Od0Pk6wt-QmveJq3 z+nRd_=uB-QRl|5|qrFLanQw9K9Fe!N=?0Z#4QzdzZEC$tM=DK!j075MGF$-wxZ;dI zt40yS`b*#iF)owbtY(Gg7eQjn2ha{Lx(Er4tlI8~W5nPxr3zWu)XGy!FX9JzXQLsg5onDR?krNhF3b-4GBNNm9$000@?Gxsz@ z?u5F%DQjz?NZQ6QGdtp5y&yOoiD8gzHpVxo0)F>P6AW}(n?0763O)(_#_P4>J@M!z zri^AgZ9MQG(8+ppNXTawsg?b0z@r3oxpA21I|5ue&_%etc`Xi4$&}FwvIs@Sw%6n$ za>M4X!Hk_AM{ZtLwl=v&wpu_$T;6o04V4J2|Kt9L&wGb&xwfh%TsK|KGL4W7`4y3s zfeYl@s{jDAMkS9>`B0h`H%e41cfohq{zKD_GS2DNLzkv0#og>%%b~@%7!rSB!O|9R7e|k+DaF1AyM}_I~O!TR(rBcklvl2PXQFI7>?UyxDfp+ zsYinr|G>4;qEkD*t4t&P(m{^*ACNnUCyO07At!pCHbOMOer1d_})wR%O9RMboa=zHQ;N zk^5ugu&*4xY;)F^$Nk_0p$Yd~YOP_P4)G6P;a@S+NHOhJ??&gTTvxP;5X6+!N_D@F zvfsSs9n^f z;~y>;7EShfl_^q9#3A;WZ5!!4j{MI-iwyuHpCEG)I#8aGnbtkbL8~4WjYwowSbYrGJ7Zo*} z2kc+S2X-);zVpHJEtW>OJp;$LIA$|J9y zsCX=1H0~QvK@o=w4|EA*f(2c^)gWe+UDvEA9a-wVi`pK9G*tP7yICuVuc-vMB$-H~>SiNTRVP1fa7dnyxHQ|NGF-^z< zQ*Ro86SB4gahuh*xLO&SS^Ul{UaxW&b}MF@js~0VD5T~&{JRcGAMs^s~(HmPar z&g?-g_^+!}jzi9WdokdJeR@X_<|3FBr`+uun$RgHz4RV{qwP+KuBXQ`VE`kD#h#VX z5be1J5_g->%6BsrUuq>OzWrX;zVu-vE#22H!5`VTOBG-Y90~C^sdJ`0Vcki0qM906 z&e-;i4=vtoH3hu07(3qG6@RQPu-p){A8WR&=v%etj>#onthPwzZ>>22d3KbxEQ_Ul zuMYCuYy~5La3Rkhmxf)^(ve#oyvapg*v zd(qBWiw^8MKR#S2^;eHG_blftmsKYEVru8)g;w;Hl{M67aa*P4bHimuLm)`>JtuGs22_UTxv-13;+^uu{ zvQB1lZ4aYA3v%3^4T2x-d!U<&JS7=`lR4!=YAX5^>b3^^JTJ?8YLoDpovpRq^Say%pFAD>5TfsuTzU4Os7^KSjtNl6 z&CygguE;ovPbs)DAYXgv-`*;5*KQ#WJBxc~?=!4Hp!j%VQ$EUf^IGqu$Y~doJ>old zeN|m-=yKwp6>ga8^J<3Pr9(ct22$)zX1i~89xP2YzwWk;OmA{svcU}RZ_I-`$rsK~ z_GtS7dh^~^Cwvn9+rE?fxiJZgzN-D4^7<-|4(Uq`1-F5M z{W+(Q^DB8Y)4-1J6l-SIV+2UN9#S6gv%vRn8I7(DH^kq0p(e*6AHP0((_UHcaDu+N zooY4^-VCrsnKYgu*rbtV;9uBQmH8%bZ^DPTE%e1!ViIBi#ag_)H&Hyi?-3xVvM@Dm#MSxm0jik1-d~Mak?84SYc@O4oDd(bmPuI1`8TwWQQNLp$rb%;aUC zTonZaLGR}ule$!mo0N>QHcHLRwIG!%o^vUu25imEg?? z1E|az;!T+CYCC-Fteqv(9k^Pm_H!GJh>ayWOTT zm(@%!i3efrSf^_i z&B(tapI&|#N%I&^Z!nDJYFn2DUf~SZ9yj?0Cm3cg{dYWm0mGQ>&s7NS5s_ulg%ue~>A`-gtqn!~NjR zAS)GYDQiq|Ub1RYbhD{zEPY<_)mXpCE?*-_WS>X6=JaFgZDm!0H*L+}k9XGPuhLTr z=BRLMC)$);f*WmxJ8gK5+z5b9*bQ2my%*43yc-BlG?)ueN+Wc@j^>^L+Q7hYVB-$x z)>=5YMu!1u#7mQk^Bh(wCqC7K2sg|u zTRGq#B)C#vFvELZyP76#Rg3R-I!-ZCTak3G{;yU7^U^Cfl8;z@rIuCrLxO}4;pMQL zk&rBCiV*B`RfugcOfVYda#vT6)K(T6xY{Ms_{QKFo`-simR5wtEK39)C9sINRh|Xq zx{oj#H?vm11n8i(l~KP@evXz6|}REOza(AjP4BKhD@V5N^RQyGsiT!)~To2o;^#D!QFpy zbG%%m&GbD<+;o2B0orZMX}A{Wf{XZ4KGDR+iNr059D(acl>3e{DO3F~t;*o#ZC_;^ z-%*fv8hFZ5-q%Vj>dt`k^*>YqhKhA;Tg$`7_#cwNMOBj+K11rTHh>eY_-A5HPFn|+q0?}XY(onKRe;m$r(|Z zz}?dP2g$2S3Bh*lLA<|~066ksLni)Q%RGtcordV#hyYAk%9NpqPH|qA#oQ!oV0kTZ z>t7W;nY7g$J)X1A6F&6&^hy8wSG33T<5tR9AmNvb)pFiHu|_X>y^nnoDDDn63@L8) zdpe(JyZ4%@9qBS&>RS6fKQ|+!5h?s+UQhsW_}{JkuZ&4F!whbcAD2_>QH9HMTa@rP zqCOS&tzk0C)n&LWW|X}fWAS#)cAZ0WAwG?c{e5l-(6%gP`31gsKV~PqB1DwYnp?Zt zwrV_vyHnFm;}?cQr6$*L_0$)%VNs{b>N91~6d)1!JBp9e`GnM(-{YKLsQ@d?aIvP% z3@#k+J0!hN7hlO&-we{9yD>{=e7BcSy&pPcH1NY~nIW}x>9UM`=;qmI0#t6+CtJs zTf{dAch_E@0nIrAwgM@QSoR#O2%ghSml<`?Z02(__9__lxzhDz_i@f>vfhs%GOthJ zG%0K4!Gqgl%3{Gwy`oJfo)vG}%U#Q(W&B$F@A#?b9|%YvZWN9U~sGf4fh z-F_Lv#mk?bbmZE140upiWUXf^a`o%{x|?5_-ur}a@#KQZH|vgKs}eplGWXa|uU8YY zMMA^`3WlUs)Fy&j?NUU`UDzH+{3i zTH&(MjDJ{pqA-|h$&VV2cz^EBzV66<`*iS z1>Tg;ITx8L)c3|W6fbCe9CfK~-~peo4%1mS_U%J2epX(t~jNLyu>G7QU zY#EDWXhgo><$F~3>|dj!pA@Daptz8g!de4Q2yKIsgQK1F8e~pZ031gQ)e-^uDA)` z_G%qz3LRbsOp}>6Rjuta$q>sj!TqvL1 z1$;8w$JY8MXsVTQgaLY7S%2!eB;l=R8g1K;X(iJ)>1K05Jp0E%nAg%}IVaR!bWQO4 zyKd(QMOH8WM8@ok8$$!my5r*(x;ee2MTb$}dNza^@}3BrUHi>>7~b^$uB+>7ZIxgW zb8Wvq^=pTi&ZJABG){E4Jp*|;oi}&|mV!?TsJ6gU^j8OuWn;?sGL5mSRo$ga z^|7_LwNo~V5wj&*<@=<-piU7odwtLC^h@L8UH5(8greAwSZ$85>lE45NKE$eb#YIw zIT9Kh&C{5@SEzF85fdKSng*XETy~*)qcu3#F>z8?FUh3SOlLvZ(M0p)rZh+nUs$k+ zcq!x>e1YP(;OM?60STN2zXsr9uYYZJ;{V{nA!WFcVrV#=B3SsTCk4#oy%wFb9X`!j z&L0~GJbLb4p`LR7B<4}H`T3_>JMp$BNhWotMeP+0FSf3CMskjg>`}#5i18Hg_sZ~5 zp4Dq6Ug0bh&t2c8=0P1QxSN4_sIUX8ip<)64Uo?yl^8>wE+SGUBwp?8`P47V*Bf^X zvgIoiKLoZasFRIPQO}7Rlg{PMkxPyfhL+ul*W>H6a$g4m^0Y7|g@w9k44cl!G$v)LCg? zwpEEQeT}J=&kr~CVKT)409g5=WYfqS zE>*tPrc9(=b_)Fq5blt;;8zFZQE9pjow**yKHHty{zdmQ`kbS1kf+iKnFZax{SyPg zi5&iu^v)Z2{!Y`%(69)isaBSjudb?e-k;`er7SxAHWyXNtr`>3UVGIH)~y`BnUx!X zL7c??#QT4X$bDE?Z)e~I1|#a+n%QLs4R9YOf>B}?>+w4o}vDL=U+3dMqBI0E=z6h#N17M9w<_a?BpHKnF_9<(?u}xwdm=I zF{x|;fPDSI0E=kycQpmE;_}irL1sM=xPemK3L`_|CX0y^t!BNhI2*6GAG=Jem1q3NvHNV}^Y0^|PTQMoy9x>y>=oyufU_9=nd2&GMv!+<|zIpJCmv zWwQyF+!7<&y_58O-YeRZxmZ<5Ro|wJP{Xd6(6KqA8&X7MF2r4$03LJ(_jpHa>XTGB zkQR4*d8Q5X-CQ@4JH9{wJLvr*XjbBoPA7qBOkxxmMx~R7v?az(j2>|}Usr@V1|iPk z2h-;8tvMTXBAc_M53Aka0+or+=RNG{|4$KhVHZf7tyD5r*5Kqyo2|e$rW+9el3J%} zvQ`e=Xf$l)uzeMHhux=yK_4~rX9(Yd#nxS_vH5$E2hZq7!1&$yQBw#mP@LrOFTjJx z+k8I&ID{@V`svA@aE(djjftEj8K!Kw#8Dg~uq&}|-I~eoqmUJ3mj)<%kNQ!{jxWrGp2z_qzw##R=LLxOUw>wUjYft1pIHFf@4N$gSwRY|P z0W}4{+A2AMz{vbgRP8lE&czQ5`o{iAXVpbX!0q*3RRAhT)@AK@cRr`@Pq;08j22JT zAZRdd!fUOCmN$5~YT;$Ja`l&gO3WWKaJHI%tmp-HH7MujbfEJ#>1+qqCN$R|@~L^v zUlxeJ|H&SE4|#C?aA%-zhIf=gFWn}BJSslK(jj4mJ>vbpmjP1TdyWTKen#jfK%s;( zG_@KIIQSijzaMv;D48SXclC$q^$A4t-qLB#tM~y}f6#1D`_bij$XuVxo^*WzdWr|* zy+x(xQh#DxZ4{t)mDi&|9rIBLiwwkI?#-?v%``!+Y0(~(LOLB6RfLH6GU{=bNsuqI+*G62qf zoO`(U2=3$l*N%J-Ya_moM@~UW_89-kGv=29fV*RUBg>97j zQa^IYl&MSDKM#M)E~!_iL?xwZl2$F6s%`KBe*8m;K)%zBA+0qtF0chjc4cDa86lhlSgbQNdB8`sgt zcZz`_Mewhy+;stVd@A?@wp6bmAeB9p6r&RuixYAu6D++o)0A)wlsO7RQn((pX`sZ^TcX~j_;ftRd^N0JWY_W=8o69 z@4rtK@f(@P#;*ET5Kng0HqwZ`f>T_^s|xNm zO6fuk$~r0(3+=Y#24Yk>0#R&4T~yF*A8=0tJ&@f$!g@N#?jbsuPjt+Tf9> z7N_bgyqF3QD@VG;p${oYfxaOVm;Op}Q)QM={R>9s+vc};>jil8n!?R0fEyDLnjE`D zIFBO1`A;=q7H`?g1a^s0#W+nAokn(`LYPODVLj~4id+ug6me3w64}^svSGS@!%Qmi zssh>1-PDnq@x|rm)lLb;6RlyXV5VhQjpne6!%e_a=Dm?7sX*@09Ix0%&4O)H_^QQe zo5Er{vW!*NoM3|vx7ocJ#QZDh^I-UA)t3V#EporZ@VVsfQ>us|+V#;d4l?~8RSvot zL7HWB)}g+{77c&cO;DzbmyyTcS*H5f>lHcebh1)YPzswVFW9FP$st`U>;$J0sp(~C zMbWPlP(23GcGMr{PM;6blyr(u-)=Cnsb|F*eUB$|t(~#av!vXchsVq|+QQt;drnN} zoR9`4byLn0iuHPDXWCVJO>H2$wF9>Z11>p9f(c&gbT&Ej*~yVcI^swx;r1W*-#7b74-I#CQeg1G1jiM3{=TIF%S)$d^BTTc z@)h&Xru(vAxYR~E5MM&M9~>qs70!z!^Q=VW%%m8;E&#AhJha_7o0BWZ^eY@#d9v_n z?#VZU%6}7YV4~<6z3`-o`@DL-BWAX#&T}V+zw+7sVep$vkB1}@oM5`Ae*p*1drQq# zs|Qs(2iTZ*y961;J7${cgk3(yZl>uS!X{9I4gOCPrFR`8Z0lN&93(@K7nN05Bj9}tcpS`N`7@}bjf$K zr;U*|@XY14SJ;XohOlY~FZ(`ueq1FQ+KHE5`tiV&>t+qY&9tKi#!G9fsj;Ubf~Cb0 z&G&plKF_joB42Ezc@4sB&FZR*4S8bH6)tR$5MgB2#xT{&Q*E#w_nm#Q!64~FWr0ulqOl296JsMg$j-Fap`uJW;F`@#%E2C4)r1=yEsbNDU8^`1#?cSr=r`kv4K6xMBO&X>wJti8i^A}6i!0fJzX0O} zqn&GcAo`0qj^6#So1W;8y)u#4;nyNyr~$lv1CiRn%y53Rhhp7r1YH$c08J>IuYd#M>xU*yJ`W z`k6YUJT`c6@zd?p{~*&N>i+?q?JQE+Xj{ScT36?hGKGaW$GzlVtY!l00u5D4#6TvT zJg>o^m{1c{ml7mp{;y@BTab<5?^1r5HYokNmT)NB3$pw#eC=hIk;cu9b#uC-jn~Pm zOi}76o!P2SI$u`Yd+O1o6hKF7!NUH&IU7Qwq5TRNNoDbGE!YI1O2MBsX5Q4B-PRJt znDnO5D)=*@pZSp$h-L>~yQ&v)^N?Ewd-NwL+1Q{#TxDn^YN0;>vUTZw1pD ziIl2++Ts(VnZAwhSK&O264Nrk7uSE7ZuzS%e0|2Nl}Ib4*Rks#yij{~)8%f>5mA<{ zWnknbm@aA}#foAf)P*TDki5x$!mIdtKXu#vm?CC)KnY3sbnSJ?RR#E~tb0^z!y0ET zL*Gic$Ili^GiE+A$ODxfT%J|RG0)3+y>Wyg8JgJp!Aoj#wj1fJ}9p#jCRYmgR2nxJP-5Vr- zf=0IJa6lRK6QIwrID!rYf24@krG`+68+e6SBpR%RGV&WZEC`x~_v;A$gxCIQ-Lz2G ziu?gNLlepR((KFw^qO0O$(@D{hk{w;TG9{NYWDP;D)Y zwb8{7AU%{H6%vo1YdtzJ^=sYp9M%x5%Fz5f-P)cyqtwVKp9p*y4e!KKL^N}~bx?E? zlx@GJ8*>oZ4ndA(>y2)+)N`f`__MY%@xyai-U@pPv?BikI8W&g%yd(Fyd7KC_nDo3 zmxxsj{c1bJqv>-A{*rSfwodVCzd$Dg>EI4P{CeJ*TY0h80c5$Yo0`TL(a73gYfU_t zIpDvq6ro`wxUbZ&g6bRNq+`9ijD^yM&_1rF2$83WEOTeF5h?-|XDpVjdWL3q+c~bq zMHD;zSa99TUwg@vbt;RTSvwH=3y9qPbiIFk_OieupULfG?%`6ff#@em{jXlNjz9!W zcqogR?}O3mRdTYzA0*ZY%=3E5euk`nhHiFON@nKCcJ}6|}44X3Rc+8<3z` z;TFw*b2nmm#Ci=px~tsV%Od(!EE1i%WKLwe3LFR?W#TXB5n2dbun#n~4>YqmZW+LM zri`p{=Jshx#>QCvTEFcxS+IpRDYAFr^S4s`1q|F#2_NjYG%c9AR-vH_uFz(xg)d!f zx<9qkUOu*iP(k$5>*o@FQuzao4-wbxidQywFJ-l^Pi+KxzuAqT9rR1BIp2GHiOlHL zYJD_3`18yD22>A9=-aMx^@wTZR$ir_pTul7=(H##4-;75T&u(h=r{F{s+`x z?(|f3^GyD&tQqy9=wN4q&N(D_7Y>%(G+*Ppnt8U{?lVmSBQQUyQjEk#8sP&qFaope z2Ly+R3Mj+U%lW9^pZ@|<7UzTvDyY#+!N(3|PnL_;cXydJC#|Vs{nUm&9$&NX4Spz+ zl2Fxn8ofv))67uB(P@+z9IlZwh!Qe;w3D!{_TP2GsEc0uWZe|&;FI=6@l;%oJDw0V z{p^{dV8#Q#B{xT@GDV`33dwWEYO+73RRr6A0aKw)HebH>fBFm11uDE+f8+K3+nL=_ z&2vP`{mn-y#+0V1qUuh2u+KUTG`)GL7FohWj~jRl4nPb-lHU^)h1)JfyQfx3yU)7r zx`Qijl&(u#Rp-WLQw4*MYATkQ1#1WKXJZm`ood-^GhKQ^_G2@4XQ5V96?0227?!8= zUk3n3hKeJh~g4jaA0a;}2sozF_326Fvv4W_GY!vh2ZTb%13`;YC&=kf92vwer} ze2j}{bEti4-id8O?^JXJ8FUpE@4ao>q9A@uLjGue<_j4ACN36k{})hu;kQEyd;asK zTY5-cJ@LL7DMjvbBU6|bjk)o1?O8y~G1|j|^R%u;xN$CfFInL#^B`zQ~f!5M1yc zSMSs9oEu%0Gs}9$_f|hVZ-4V9w_VKD^2_zy-N9vI?^;fmP!o&3qH~Wi|7patkzPW7 zgx4^QkhOpvM5Yu`34gSW)<`wIILluLncc{wzZEb@Xp$BR%lLhgcyw70Bh8A}9DX}8 zeEqIDsaf+^(esCJ>4&FFrdD$gjZzFgCs8}NEM@N9V|w_r$mQf2Ry0ZLO~uNV8>Jy8 z0vkmPr_Q8BMt)ji0~B64I~m{TT~xW#q^Dg%tKA7DF%R&}LO%C_M{ZNnHSezYZXqFR zo$SlW>nc>-cb?=J^7XI#2%0sC zbK{I7zjz&y<)=m7M4R-Kt6S+M_bYmq|?TuGx?5HFun~JpXMr z&c)`z&PLny@q>BuU9}EJe<Qb9{gZsjLV{2C3$J1JBV|FaAsJo-I2+%Ig`}J}3rYRpu>BwEFEft;S75vp1lx!L)@v9KREP`PbJW!?_~jD?!K6&pTc!RBh5M?R`>iF`Wpr-XlBjwrz6qSbjAd+_h~AyaOK0+&`Y=)*>5u z2U@%V-kRqeC42u=Nx(N>**a(0MiHbK#XQgP3{iM&9BZ1yfU@Ho^6d3T!1m-cG4wg5 z2U=sVkcu28L9u@UE7vO_hu1sq&;#?+SV5hqOwW#Z4qU%qaVcn}&_AP75HAk~AvdA} z`J?Ub(Dql15E)c4I_GJEY;*G}-og>?{^du%+t+O&&&)f?_?Ev=p~!gio|#I$u9g=< z70l4WWU?Lv@uw0EeBkTnbozcUD?qI9F4nY}|K-Q9=O^5Jwl;IEW_@5&QSrFm;M93> z?)zzSTBj4}hHx*Be71;Yzz;BSH8+cKM8jSpdEA;mkIjQK<qOWMqO8oo61|NBk#piIkD{qD2HyM9K&)7AM*F9A;l|)H zDAz~3_9l?2t$jpB-H?TAugeHLyp?5h0OxjZIU@Ln%E|SvlP4wY3?fsGnKHYs$k+GY zsRHY7J~6}=Osbn1I@u!z>03!2GSP@%cm{n9NnDfj9u$CPnkwHYAPv4~52F(@-* zUf~tisAC)Y*lhH9m=V2`P;BdA{9k}_my(89%655J5*_oYWT`l95t2PI^z{DT>S1V; zx~u8%br5Z*mbGqxRp(1)22V3eV(&;z++2wu$=6TWUJC9RV%3oKqF&*`8Qe4vM-V)Q zTs3*`RzIhY_GYE7!!yxl16-Gl@Wwv@>67<;lA zspyZ6m~{ys*(R28fBY|}3Kcz@#>`(Rm-;xj3ZrQc;{27`ip`hJ6YrS|q z%r51>1E!NB87XSss@&nJ?`USJVU1|JqIn@};q=a|2)1(1R@ENkoO z*;%t1#}Wn&|A7YlNQ##_SHmR=vF*Q=jBJpYW~g|_`RoZBCqssll%q+{iy-b#(4Wp$ z>|dz*?>DWhjTUl6RQR>`U~dOqG{>$64G@sDYo!?wTT zLu}0ZC#&)27)~Ax-g;c5a^H8QEeI1)_NQcM1{4suntX_)v59-i1$2D&5amd(;<*np`&rxPAa1uRfjZ&E`w`+o2bRh3k7#lU8HSO#T7MfVSh0%pkX*EYHYO z>|HPYA&=E}>oNEs zUNP+7)dtgb*q;g$pMpr#VGPW6kh#X*JEa|_su1V+(hPkan1Rm0r0;T7UauzV8C!nA zKx=3HVnEFA4v$XomgqEZcQ-SA13RDP*VuR^O(UC(uqmhAg>kJLE^vHRr`Lvgn(RxV z!Q%R3r(h83S9VeJ!LDro#%t$SEV5)u{7c92>GE_>3g?xV9&Z%>}`vGn78 zc}QOu?lUyJ(`H;{lKUpBUBI)-DWk))uJ1tOb(n|sGa_>y@WvN8byZxpqPL#H!(yY? zp@U8hb?amiIQ$eL7F9&MAb#*y`vtpN;#h;E4a>r|)#=AkRIwc~N+yDScZ~opy|r-9 zPfet449Ss~V_{-UW3us4tt$VtT*QTA4mHc6TaCQQDTH>jywHXB+XXa` zRdrZ>Qq65zl$nbP#7h-}cXme&ANT7PMpqWSCug0j+S{z!qGo=ABE*CTelX0ylX)$n z-)8b8_-Ek>VN9kLZ23EkGRXY5`qC#~`wQ{@!=9GMQ4?L$%nk6R&tNNBL`M8Ag7l>T zP@F=+JvG&0j6WSR%kN}8mk|jXc%p_E-kQ%?ejUmUNoJ z>4hFXJRKH&%9S;QF=<#Gd#qdmkCP=M9xz*s-qC7$+V;oV@LT&Ee0E8z4zuI{!c2@^ z;{sa+siz-;B8{8ARgESw_`}`N+!CB0pGA9xAlvz|$y`2-_Ey4UqxaR(c>ee-HR|+= z@AC(Cj_eiK8l50sfvWNM&dvSMQs$O2D;?A!qdlpVTI|U9I4c#X(^@SBYV1xY7MO?C z$LXa1#QsHhz?EA$^kTBZS3}k_{Wi2$%2<<#VGq6(qZ6RYo@2idQ52hDYF*SPXqLjK z0%5C2wmH{mDo$?X(}p}B)MyVfZ8@H7xsqC$%;z_{#U9k;lYFm75sgDeDy;7%&|yC; zsqm@WQk-cXj&eO<%;`IP{==_5pH+NfNP2-Bs>ZMY;7u=$_}*Bs{?TWnUdf+!hDvPy zmAYqQI_b6l3yQUEMz;D@r*o8Qars8|CxWI9PpWmcY0eKd*1-NRy?q;E>$M*bGy$3R z*L97Bx?PjbR)a&GS$aT6Z5g=~6hzrwBGMIV$Yn8VkG^%0 z>00w`gLWd>pMLg^mwwgWymhZWCQu6adi*SZYgr)|u5RP>L=O?2>29qEdei;kaWJoc zG3Y~e!UulAbeRszN%WTgd*ahp=H)a*?4 z^p$~+Hk`VgXBgfUAu==jvwXL6$m$-s&dN1Q6k_xgkUHcv^BS4AF~JO}g|)*?3n0F` zuj_lT$DalIx1k3Cj6vdjhl?qM1nK9E?8p=$qnx~qh%-qR^CYfjm;9y9nk2@HQ|h&Y z6OAgtm&vhgr3|#qXyujZi9w+QhWy*um5&OY!K_WDzCuDUo9vch0F_8~+k;8obnG4Z zo0?#Z-YB^2f+2dN;5Zc|!HJxLyN%AcJ;<0ZY0zK>ZK_d&nth}9=Z=A5hEIv2@4H9~ z&DniW^2^g8Z2UgHvMgV1$Go!qrrM6U8~zbEis^=bHd!Q+fD{W%!6>L22hxT48c-&4 zMBVKPz%kBlcr20NXT#Rx+#pR7hBHKc-^#LfwPVHa!>==qIKK}I4C{pd(@y^xzhpGD zg)+^pKe5cW#A&)8$gmDNkCI_HbviFul-x?&e=8-qy*sz&aadAq`=18taXkKi8;EFZ zq4ISb^eFQ!wJNR~qU6(+On;;q0eLK((l_&jPuMKT`lw4f^hH5?`6mvFj}IRxcnsHH z<@lF;Zq0Tm15X4?EZxX>n5%Aa9M?VMGY12w?9IyOPd?>0P4&1>?(Sc8EGbErxyI^Qm$Q*&-+c-0&^m*ryQbWYdgSR3ofA5& zxzMkls;w8F6NYLswX)VYmWQVWrO8OE9q%-neD6HiQJP^+s+J44wk{+5-Y@k2dZCf_ zV~OOW<*p=}lwEeF?7x6Gy8`IXuUhw5KlsTzQ624w&86n$WMmoo*l9dE_=GS7E87{4lWacCy!wq?=>Qu5Z-s0 znh}@;Wu+_g=?Zr+{8 zp4ph*LD*O__P!mmn~?OmN186%Ba=5&yv)6Fy(~1%W(1uxg@>{>HM}{>=Tn_dIAGt@ z&DqheIgxWqfluxTF0p8x{^>CG@~f>D*VMoJ?&+`2`#4x4f7gqJZgH0!FI#C*wrCqN zpteUYa{hHr9^KlMdi+F2n!(|lNQ#umfHLOaF+2Fix-@3q2}S+;LQQdR0zp(Ce}UqN z_L54^6*PG8)Y^q?EMfRBpudDGVtgq0Pvw55+`7#nv*S0GgF@-Le^fgdJBl;hv`4i& z5BIwpp2_dEYMf{0HoBq<2TIW9uDbUmUB|9Nft8rmipfCchXn|=v1 zy0-pq51;|mdcTOOF9&tU?qIlWrRA6 zbv4e*gY3whFT%j_O;gvhBFxP+7i;I*z&_L0QPnCaPtZ@V4VwL8C*9ui{VbP_2Q{gOzW>J%h@+fa`fWm zJBtfKEcDlZi?(My`*XVWW~X8M4pc{u*2FOp;!wDv8x4h8@ov6jcAn<+Z)7!^E6yD9 zZf3t{y7r|sXwZV4zeZuAkz|~ny|vy03%^X6yxz}$q9+-IaTqu)BW99N9VuZ`%4}`9 zPY;*cy2_W3aUAch)eltH{O@nxObiwK4XCki&WgdUj| zHY+-z#=F+cgxyJy?$OIn;M~CmTTrawn5%$%s-0WypOrK5_#=CKVa(M$@-9^W$gP76 zN`w8F$a|m1vEHX}g75fs%E1M)ar13TyB$+N)`R3AqH6>gqSASl?KF0yn0^V8txBeo z*bjAbKSy?Vc`q0%WKA^;Ci^?NL44rs z1sK)(l37J1%+Pnk+-JI~6`RXOrUZJqS81HRJ72R%0sl+x2a5V^05zR;xo<(88sMn} z2cI#?lH}}NrZ;~YKR!9upi{_YPBknx+js1>a&793>i1iHF)UJEe5`YWS5n#a{=<7V zGjdQSZC<2lM=MlUZ2>D%ZhP-t+Yr69{&gL1!T1Bl@X?EjK~ssHu=rKkOm2WapnrBjx&Z3{Z{^FJ{WL{38<;{Iy;{;p+7af96Mnflp)A zWHrwibqI8x{`z=K#cY%L_zS<=o=I9{eX`bXyv#7=flsjD!B%|Ljug;a?-`ovJgVFI zA^Z!zUftS(0bi^(!t?-az*+`>h<=rN2hR?*w{?0;qHbsvN15|3@&xra@(^gnkcY9i^$AJF+IZ64?@lYW@P46yQcW8@e2vE$uZ@ z*8%-H9C?ZO1TkY(=h+OVE?l@cPv7&j)&`d@FXn~Z-$CnvfxbIdU8iqify1m%aPpahklD)khSC3ZeQechbKz^(<>Oo~Pt2+9 zO2Z?9g9P|N`5?;cXkGa}hnl#w+GegSdq_YlN3z`3&DYb2vzV5KySP{FiQYemCcBI) z`$PU#2!+;&=C3<<<jSYYwgpni6;-4#Dl4>hrtPs2JuAQo4q$%d33AfLMmojan z8gHch-;ZLvkz$;tke~`|rVY2_nQg@arMa*P-uyotN~u)R@D4FhW1-i^u8h(g;MmD< zQK-;n1ko>kJdJTDANSF;#;8H&%8mU8Qfe78FaxnKemqK%u~sXl2@nn)s%u$Lak=^+ zU)C*9*X?z}<-s|$l(}_zey9H;&UvEvEC+WX;kx-TBfq9Xg(1b98u5%&!E5<0nQL_7 z{9I6*su^wC7{Fmn0`Lp^8WI1dIG&@|U<|fpYaGn4Kfi@Nd(78%Y^w#t3Tkm;y3dp5d* zOu?m$XiCi=kuuh~@Y8@5)bZ5ate7)40VPGnJGPnZ8&t88@_W@OKqJtBWaB&hGch3Y zl030KSk}>Db*DXH*3k9DnAtwt_MY2ZpAE_A=gP4C`}3osMhCBfggC9fXTLvdkAZd!8zU+8APGSXY9SBOD~_8FkuZNCY1pRZb?d%ypLQ zWWVAWTZssY!untoH1>FsFDyLwCTa07X>Mtfoy{wC6bQsu9L5rNDMw_v_a~ZfKtiDZ z`SZuhghz|;P!pDfIffJ?&R&=*AY{sj!*<)^-A~)nq4HKkYKG$9n_OCP}5x>S}o)>qb?f=V>tX z*NC%vG1Q@x)9;$Ae9oXA_kb98Q{-MdKYajD2O?l>`AHtoTA6uP#zeo zD_H&}CvFu3f0acwbEgxW)a^aiVX?t+iSH%TUTnBaOK#YlZgiaPJQC$;*vE3b%r!oN zNy5BQ6FJOzPql*K?{R`C?i#_l8~fU~tC+oYf``b9-|w_5r-+-~?D26-_wOyF zHr~ED<-|99<3iG0W&6B#mT1N7-7!qNeG9@~53DGEu17~1F=4Rn0}9nu)|}LMMoNZt zgC5=Qw|^~9F92b?HosU114r;IIoF8|ZU3@jtrm+)!0L=+VAo|>+mg%0|EfEsMt0w$ zQNBL4Blgq=T$dzhuMT(Uy$!NnHvQN4A{&N-ySVfX2yg51`?mE>j<>T+u4(xSb$r02 z^OlIm3vHd5lty1QDWby6xXu{mnLkh~_YwVzxXL#q1z&HZezz>l)3J;d%-GQEBduH) zZy7PWm?=@(c0tXoF_6w}_Z`JB zvM7#d?zC+|O(AXw+&#g8-oObrrCc?ndkJxNr4x0*W-3FM3+~b-8iQ3}vqj52I9KTM z9}2T$Xl)n^@7sG-Q!LW7Cg)Fg9irp*tBwk)%=j7GFQGRO6g~~K7fLmdMH4z=sf~%n z^@n8-lbe$nl&Y)6j&CN$!hAqqy2m;M2sUT*qT)GVXBVswmnOS=*Mn9CewZGIfE9V8 zF7;F!O@!;}ZxLG^TW2h$Jf@QTiVT|F2V-&!V|DVQ_9I;f$1-rnEHFC&Y7b5hyf0oQ zK6bCZr8lB|)FivXn#{+05|=!NGj)ljC3R{(v>OmQd6w%2+N|BhHObB=Ie1LLFKm>3 z%`7abowetRi@6Tl>v^yTOD}c1J^&4Ek9;p5`HdkbtMWT>A9Y~N>ddOGFxqiEW6qXG z%&Lqsu3`6e2ho4kc(wm+$85a@%Q{{kl(MeU>NuPs=-qcN-&%eed|_es7yz}TE@HNc z(7`9T-1n0ZTRdY+Lj5 z;5Sc!evH^nXJvqGKkp{WsL~~ON92Zm`+();lyu5M9eBg;QDCL}t2$SXzHps&ePdb- zFIUPoZrt26Cs$!(b_&6VRnz>dFDLEtj!~j$>jm?U?kjCh;T3@lExd^c_*}ABCp?}? zE07JoOTkyed*zC?OkMiVwr|a&oRxHKnbHC6X^U!cH70LINJTfG-GrUI%&oHqJ@@q) z$Q)6>vMb-OA<279jbL1M#5(7$eIkHzPYm}Q7xXGE7EqYitvU?>RL{9%bja&g?z-)C z^8Y>&Cn~mdCARlSMt_C77hZo#NRyQ7-{^)oT@)fSyx&+9;CMXjsO7) z4TW4urr~wCTkwMRcBC^DtF>Tu#@=6Qa%AqyMV56;00!qpWmCIqVkKZbK_< z1G5W+AZ|~?yn6Y#DOxPW{I#@Y4y;KDZ;sq^#Y$jIINf6odos2nK2LecE}^oPmBqCl zZL)3uXWDWPy+Uw~>iI9Ba1QmGRvB_a?$iQX46c&8MO@0h zjti&}P)ZVZ_zmH}@7(IFqW`b$|i}gP3=!o5pnHCRa zRL!L>x0VOAioL;}X?^xd+WIwGdO<4feEbq}wf8_lqThJ2tXqjbc_77IoRfZ$@LoQ~i?e zr1?@XMIqol!nRMvHd1(iN=6PBEebfw%Uw<;D=!@vo{^VAIX&{V2w8m~{ns+w- zV{5#FtreQ?n@GOk(`F0nT)gZD4g+iYc~7;Ff*d8tw&BjFE0Jj{XYL^EnPqmwJ{co| zdOezks&TW6!-s`$+1{e!HLh7KBNo^27c*Fq)iUOy@8Oh2nL=QdsxKuOUjF$W?|6VW ze>qzQJt>u2AXGhO=dv~6{v~Zl71-&V!EId%#4}I@(P?8t7m>JZuxKzNnnTNjOnsGz31WRMYMG_XHIAvy7j7 z3P~xi%jV$t5eav0+-k>N*k;mZ>kA^}upQ57FkC5H#HO$K?g9mH>gE#7#DCMR(vK2KLKa0cy7D-D2;Hd!7Ovab33!4-ao}2>z|&vjO55I?@-v^zFJh`p7svQT+;Unc4KG$H; z>%wYS;1B)!@XW*G#^*m6b1L;nz_E<8gaxS+5hG-#`+E;w3WD=3%!&HC)*HK0?!6&e zOQ1D8Ta;#a!!pr7P+GIXj4sJaA~%7e1E1(Km1R#b=f|E%jKfamvVZ{fA*RqTi(Py% zMpQ)t)O{EuOy+jTi&4j6m;=8&q3j)vO*rQZ-u=ocvvQ1+s@ZnDY!>>mTpC)ejzeZL zWE^zy2ddOQu8iaEIB3JTkL6AKH;PGW#dWke=^F}u;_bVWF)jT}Lw7x(zVY5yo%R!L zxaSAqLz{iRp2Ab&FA+6`8rC{1sSylmP8K&BvR3RyslE|E+9c_1zm0MS6T(zLRoqq1 z`Qf~Cq$t#YrD2GaH_i4_VG$`0VMy(TGON&HG^>!9Q0lPL9-6PD(?X0u-T|To$m23i zHd<@qbqVLq$J#ZXKRf6Wv-yNt(i-X;%f<-o;>snK9;JT&G&DR{N%q4Ev(lZH^oexd z__yo47Cdh}kyj*(=b3h$tWmvzbNS8}aX5+G4R}ZGEh0KIfb~4F5CMqnkS7GO!gKGcPkwu@uuk5I&j>$ z?2_>&xC1=3e7k1&OBD;+sRKGy3*?X@s@MJ&5qSR<#DKGc^BY8mvjlU~QgN?W_thLy zgqW7`PpJH)YvLRcntTp?xke^)`&_d3**`{el&C^w7LJ4__5^|)>-Os$ToH>My`T!~F6 zb?hBd-vYPl0_V_|%tkd*&-|+-;>EpJYg(mow>4`xiAC72z4f?uwTa&=iGOD@JzE03 z_bAw7*SS$mhww_{Awop_82|K2BEZAKL&N-&WcsU=5(5(*i=3T|O;yb#9}J64@hNee zLrlGglGD`Xd#$*N^S7&if_GnJ0w``*&y-D1%Kpy%t&|Op>>IM81Zq@5IdVc7(ocGN8$9}au3o_2DXmFk`Ev#n zc2I)U^m5flED{nLjLDfvhPZzwBa*S1HjeX+nuzDWbEP3K5~g&NmENlAs+ z9z*2Zw&BE$yhT{2MC|C+4c zwY#i6Fh?+S>W}hNL7ECphEy2|a!p~%B>G@t>w0parJNBxXciX*u-Y^t=FHqfbY}j2 zz%bw~>h$qW+qk=+?e!J!X%EyLDDj)E32(sc6OE)q2ux8Qm!6c(CZb{}4_;FIPcFmM zgh7FAOFaZe!kjkimQ&rNstt7LA|kmmUPdXK{s~2e@)XxM?9s@ZCba&di3)|4qKWdt z_sD1Gk;bv;eUYwT%Q3+LTAafEGA&0Vs9zhaM)07r?uo?sx|nkB||HVv*szNEuOM*^c`Fn zewuwp6jp${l^{n+A0Qm3+JoUHL2oUJr}tOtYO>BjMp^MSa=WV=Dml+JkQu305$ZVF z@JaD4U?n|3KQI$I$q`1zPV(FFTuDhuB!CC^Q~0}Dcoqh1@m~{_w$1~*jL?9OTe)!K z6I$Q|MJS01xuX_5UaGGVO%CXxpPiO9(Yyc^B$wP8!W*__ejW2Fk3JRo7q^<9*AFVf z+b1N*YU^rrJ|rx}EJ*9A(*J=<+1&S}>)HOk2p4!T6IRbbWe z`f;qtviBsr2=lc`>wdE^UbR&B?sTI>rLfSShYaEV4gO|fU@CXu;pJizFw?`JBQ~j> zI@^Jvj!rK4kx@aL^L^sI5(8QI`(G}SdwGC|G=f{{;b_aTe3jJ)_28Q47y6zhfB%%N zGm)-QXHQaN^Y_Hzi_22NIoGoYNx7CwKJ*U`;YsldnNmg%yj*e!GCp;YVnun=T(%4a z6$}L>n$jaEVqZcSu=-z9oxOZs7`85MA2fT`56>!t+ozVb5xjEiV!mt$d^v1hR)X(4 zJu8JVxqnbCCvSk<#`b`jQWNUYSy`pFGkh6=%$^(%U?N6m^HtLI%#_Y zf5`pu!EZ=`)Yw8~qUJd+%0JWOo*3JfYK7wulnU%TCW*!oD*6GUX|0elj-astsM5t8ifSLA%WbL4$H%bLJ5oA4pI?@8u}zv-r4P>TDOJ&Ub||I=(+5gfTI29e?=-696$JLs1YqBP?tG+5~(hXDbWA@ z(UK<>DcLn5qEhUY2hnil_jP!XV^A`3s*?Y(lp2*d%m<*NV&Zb=x7vvRs^Tj&Gv!VQ zSrzhSeDG(Vs_3?wC3>#>-yrB}s)YYub+<;+xt-|wKQTjyW*fJYLOx+2ncT8y)v9)oDuR9|`7EB)$@7|$mVO$_{h2Qki7KhPKj7hJM#9|^GB zypaSKMBO}aU&9xR5V@0`D;3?6Fpa~wn4fm780r)gruqG zYJ?=%&=+W72;*8t!IqzEgnHu4v@sl27J%ehwOyYW41(p7C}pfoOe7bux$@J(6^}-} z^|5Pm;Zl)i^ONj?Lk41doUdi zH@8*>C&=B1qIu|LuN6fQX=?p}Htb{1Qmcx*e)CdYR2RJ)_k9-Af!s~mj zS7PnQlP6;wU(FMsMkVqUFW7w?E#}yZ%w&k9qsQ%fd$3y zSf9lpG@lIv3i_|0Fzh=x2(AAQ(TB#sgh3}~$0DO(Q-$RaGx>xq?(#hmLh1AWL+YXb zL+Z;J*wa-&O2x3Fv9+UNc-zWz>MN4-Ogo3v>Q$!#V=L2bMu_dHey0NYafO-K8Cr4* zZAKD%UGf=ztO?fyWb51v+NrhU<*Gj-g6X(81P>!jggL%G*442levN3WI*k38n!{CQ z_N4<~pMqo_kP8m{XZ(qksAL2{&xCmg8f7CuM0tky_Hhbn2PDfPy2E;D1z8PrdMb@* z$gD0VW#=im!l06_57jW0zsSn!?y+pZq$MS!xXNNX!iD+Kf`=|>wxgOQp>;R0%ggScc_pdSuzl=MV z)g3N_xZ*8~Ck624i z?mIOYJ&ws`ap^1ntmeGb7)YgF_qJ85#70(83Td9un2T=Lsf(#3D$lDQc*tUHa_(B% zN;}UIZ2AMW93OQ(pR%O2km^BJPCHuXvzF+}B01cO*WX4x+(uz@o~@t@z7;uTFtuax zH-cQgVB3)6H{AiFdod(!3M(J!sOnP!?M=UoVst=6rkDqEXd8AU+@dwu&Vu+7DY@N9 zQAZCX7ZgEWxff@^4TbXPc^0d6A31K1!|k+g>i7|}8!4V>*J8_q-77h+;CwuB{k<_c zl{hAgTJxNAga_QBi8rq`$$qOL_~wTamLSUh#?xa)x}r zj98H*g2Ae^Cyt$M@R8P3@|+NL=nNfj;1{c~5Wlm9@4WbMLf1r(?ymF|>^M-Q*Ner;RN z|DT+YG8r`1{*^7teb%a?R(-&eR+U@+>w7qo>0~ZtpSMn$=!jV=>}l==-S3nDx&Jg2 z_EF>Cn9Ris=8|IxX+YMjyQI0nSdqb427-oZGrd4Oye$gG8O>}3mZOLNm?7inte#Mo zf!9I5`{zowhskWo>Jhsez2A`Dsvdzo*Zb!aZ&LHqp8H6fc-d-aVwvthe!FO<0S`jD zElV3gnqXW$rKE%Y?5F|{%_%etQ-&&#dvmhM$jJOYJ-^dL?D?nwx%s@&el?`~b$+Lb zLP|z#*DC&t4U(!I6&lr4P4o)sE@#YgZjL&Tam*wcP5-iVRaEHDMM;a`O0*vv>2|^B z@Uo|WqO@|iFBs^=Ls3Dmkfu?|^zz?^oFui}YT0TTN9rcAn~|N1bL6RfxWI5UAcwi2;uCjb@-3!!Tu%jHtz>=+}v;;7wyJ zprqiaW?|`*aIq;0nPt$Z7EIDlPF)sjc4NReOxOb!Yr2 zq>R2M=el{DIxAgPuFJI$8TC(j!OW*GOZKwB=N^l!NtmJQLfH$Anq;d@9E=={9Fiv5 zS@bl1H=$C2-FT2QpsQRLBb#c@Egq+8BP{D?O3!x}?cmnemF=T<>_xLzuyb^i0_iGC za+!9Jj85*TSJ#3 ziJl^fDG^69<)F-Dhe=XHxIJA!J+*1wNJ;T>eB2-H`*697i;L>}a@E8+7djQyecSeU zgdQP&rW5V7(kokr-$~CDo4z5wy1Y+H4Eph~IYjA$HU-75AP?QY!0MlwXMB1RgSM6k}Pz0#<-xbYv9_qN7He)=CW0HsAqv$k>NXC==dd0ttUI=MeU^U-Y; zhQtvtGxMq6I9u#(iT+|N<2C+G2BC&Z5H%Vw?_gn&kr5#z5u!-r-)9*xU@;-okU|Wd z45C9r)x;$+37e8l+|})Sat|2p(`Pl){DN94?%-`46*KoB3H8FOFTDwM*VO-|Ujtqg zqF>`0{*is4q!7JTqH}+g`jSPJ)yFl%Z{WYJdN0)_xo*)V3x;lL2IMiIXpwY|+9sLr z^RnGFUa8GET=|gbyLif7+MZ& zwUX#S?K#ExR8hBaD=Adw8g8!rmABG9%a*|*d@2(z~tTmgA5$Yv3Pp7$T?g6#V2oyyn`?oa80Q z0+W{9LyPAET%?huWZe#9jChrN$H@x6|F0JG_*$s{sl}L(Hn%p$a}*nQ4%1l$q=r&_ z86KtVe;Xh!G~LN6syzH2+wi@Y?6KrU7|4?rvmsI1W8DH@@uB0~Zdg36c?54gLF1v& z%0riCtS@wa?@raQ-JM8$Qsm_SdWy|1syW*l%a^;BnwX= zzro6AC(}AZYZ5+sM#ceR1?1hLa|8G6iiMe-EyCvV&qrvhC%*jpsx{|G;1xvw*@{0< z9YTqB@W095WNCF?A6+)B-egABP9g>?ed0wIb|7z|WI^+G`I*~aYol!+@x%P(^&A@V}n;)-X5yec?nb1#&o-sQQW=l5p}4^t`)Y? z{gE%L!rD#c8Rp@+FocyU!E}aymgb`ozMBu$5$dZKo9LdaNBpL1`zCJhzQ?$;u#Bx} zt?6uQ(pGBPEp^g}#lCb>l5)$82HAnW_kEU^B|yOFOIbx7{(DZI*@GH8ID3l{znZ#q z@vCf3cVizxNj49#8kW^8Dq-xj@7K(6b7RsZaOc4%NtfmBkk+0;sk7I}MYFn7ZGsqa zcl)@}k7omCvu5s|YJ+>M=E3@TPyyTz!uS84{RhK+OIGkVYJ7g#GDSlJJna1ee!VQo z)rr-){5yp6d)?5J#!FV1%;DTmK19ekqI+r{@$2gDQsbl7jfo~GemGF2^2+k%PySldE17LP0M92#H|-A(77d(uBtXy(vmN)vMy-wN$P zA*(|nzeo6x*b%7xBmw3Od6*{GXR9vKrg@cgb(;JC^Hl6nr`uZEv4XA@7GDm9w^^*I z$QGhtPa9Beim?nQ8A|Unf9(KQC-`2(Q$~c%{ntLz!3kiiNi$Mrga8_U5Zuw~Hl$Y@ zEtCq>abf85!#8`&#<)M`MWnN*y9^cUo)%3nuWX?#dqGsLoCdC;Epr!6ngs=+FO4mY zwTbVr+=cepL<}#>-c~?!NPDExd~Y$4M9afy{g#F@dCa{LuSgQCxRM++@8Om3EO%Qc zVfL*8dj&%Y)NBZx)*xl#uQJOSPY3@d3Et_0y=V{w7vc>B5zuohj zBmIK02@n;+EYsiH$T4!}hsXGs0hGW7*uheW(cvHjD*?74BxfkTt_CQafHBw4Utxf0 zjFQATzw?`OShqH9&C`QfeL#G9=lER~U;~J3saS#A1%ffonN}=LCZZF{z^eGSv5zxj z3EiKZClET*bpxet^}?{JT6F!$ytt3OO5J1*Ota;T?Cd9BRXm9~m{g z-0DT7Y~XO0Wih=$CN-c zae&^O3<158`Oyw)L=x&}&(P91o&I=-#rYPG61K)V+^0#z6{iy`*BnpHPrTx0AAkzJ zuNB^7mBnt^^O{g@JMIZzR_Zn4QkII@BTlvJc-A||GYK~XE^v<4e9O78jp*+{lG@=og2H(x+`DfWD}+okm;%K9P%_M#?>(y6hwYxODd(OK6zV1B$m-TEv`# zw&8&81qTYcN`So6yY{4yjls0o9`)E0TpL?=rD<8mty~D-7P$LvZ$u{cGS9IcX5^cH zuwV&*@^SL>gotsjZEQjNS?kk{Zr3OBd}woUTY@hrHi_rH0hl=fRcUYUR8Sjhz;Ovf5k{Is@5Iz0xHR_-a2y=2*FAsU0XpDkiBh9~F^dmAB34hA!#Mfj z4orJ>`DqR$DJtuP$_L@)k$GMset>Nj2_bN0)%VlRUTJ!$7Hsrcube#Br24*4jzaPV z*aECHY>sf(VYxr`WjklJIG$^-?0C^8G_0{Vz^v}sfIE__{g^i0X=)EcqoZ1wuGdc_ z#D}ccSU-dkC28fi%^zvr7JEQrG~o(nPvTH>cuOvbLT{L6W5ux(2_y^a#omY!uB%6) z^peL&9at%rUq69Nqb6qu?f3;|Ii11)T-avrObBM*im}zaxf<-sJUkh-D;dz7&hPPZ z5<3#kHkO~GNHW&{Kz-8+l$1S6wYp2qtt!N}H-TN$<&G}~C+lJ5(iPR&A0R8`zc7kc zpd=6G67HC>G_^4D7H9+{-oT9L5lqW6@xg9X0~^mHZ%_#{E9r zu85{8LfcDUtp~lv4d(#o537Z81xZcE{qLvJGBDcF7U0LMN^%a8>|tRly)!Xo`>?!b zO{j%6N~$eSqGnQ9ooW1Z;KUy87yAS6O zO-&^l#D3Yz$S%LIgmo0YPxJ3L4wgv_-?<#PObE=AwR;`&?I}($7LAw?)jGCh z+BO)17|PU$BB@WZE1DG$8xR~d7KnUf>yHc4lR6+Cg+dLU%0x+y{;`UqkMC++M1 zjE_Bpq%>QjmK89})IE_G+kh!(1Fy9!Pv=@It9K?0zLcD~h`iwn$2vfgt_ z_;NXw$IH2vK;nP)(@HK6EeFMmOMsaMhp*r?oeM}2h1ZS(8gk{ct9mqgpx<8(R)@3} zK>sn@Aj;i}Z#^i^^Qn&{C@1 z?xB$tf9TBl{p*dR!WhV{!eIt=z1NdF%1(QB{@A%rZR7$jFd(ofz%Dj^9YR&j9;FXj zmR`Y6AP!dd0K1sz^<2O%#iP|hi;G|Vtt!gZuOY)NB2n&`g#ZY>7MNHUQfK*?8RBE-@Yjn*7+^=cw>S0xrE1I#{ zNkV&+z;D>qx=9TbqJ4m2>}}8-lqOnpMXISMu=E+Vs#NvJUFVtWiDnB_^GF>+Vt3MG zOa7dx%&xf*x$D>q+A#CgTT?f0)BniL>e}dAm-q0t@tKtJzYtnq3LhmF3MFx5({|6* zYo!%C`vD|L>tFdVLGAxckcH&nfF!){d;oohsNCMqI;BfYREZUmHKfdgXUZ?UYi zOB>ycD~TI_S5sCy;9i6WS+Fl<&i~}ZP?**8+(PJfETkg+7 zZ9%b5L_EmnS}&7sg^~F`8HLgf<;4npv8Vat!7-tH3|h5y4TvKw2U-{(QYQi&DT@0t zfoga82Lx+E=%DpvY2S-9K%;WFBvTTbwhGMzDcstLjAG-1b>F+5IR(4oq@`oxas~O6 zt8Y0H9MgwW#nbeA6=AGu|_h#uJ>vr*GX ztm!6O)gZ|KtT`kSDt0P8XMRfBb)y74&t3S6+&Fk$_RG2}AxbWTOIu(OeAO7=`5y4x z0%v2_*Wo6)#;K0%e}88mLRi}cD(N>^nUUu~0HPJ#v0i1~6|+ro+mIf;Gd8twUSKd0 z@>(u{(>0Cy&7+rj5s!&CV;>u11v6`{%_$S%MG>16qA%e>FoiZ9%U6#Cb;b;0gGKP-v-uRdL#Ma3hRe} zlV=HOV!m=#t=pJF@ayf8fl|7NhvRu9S?;kN`tt2w6BM)~JMl0;!C!VV)t{aUgh=L> zM64k?H*7EB0it~??fIk_wih8^B`e$FBO1MMI{`D7znJW9w2ey=V_ym?V(S$MPZlwR zh4mV=@ULUyn8gJ5JP4qem&C!nKL>A7kDN_Bhz;;)6M6e(gZ6G%lqvq#iHV?^=QhZZ z9N9C-kCpml?e5~}v6W@y7+n!4t6MJ40A2r$ zj+J&1IR>X9*GD&mtLwu=)N+t%n=C;ly3+^H#4oT&5<+H12tHk9wX>A9(NBYw5+*;l zh1gwg3tq?5X0)=Azub-I<|fxcW6xxUSrdZiDad`N@Aqs0X;i z$gYO`<3PnU`5&leKRE0lT{CxumepcK!swD&=e6BP)$}Q*iRf=bSVE5&+9}ZVAXkRpiK~M2G1wN4?`HAyUgy(p13}vw%4(t5fPxo z@;FMKX8c>9BwZ@&yiQkP=Kf;(lJW(I0p$uBAgl^WsM?jtqVZ{!jrTb?x? zF}gocL6!D5!C(T>zOgvps34?ocQ}^_fR1PG&#?WzC@5To&$f@j@_xBe-+63g@x2?h zbuDOfrSb^)33m~Y8lGby+|v!gNm9K;d=|h<7m1Os~U$-NXsBbP5^gnDwRQ^Xk@&(GrulI=G%Fa(pdnDRP@3* z*~ZH*4*^qWEvKSZ4+b8&^)D3&M2>_pJK!z#nJ5zc8zG?{%Zv$ zk6~m+Hf$a$LJWp*+okwua-7v`7su{m?aX{SJv$lrBv9=qJN56OzvFH2fQmA6+53=@kQ=V zMx_^nB$0iYTWxUcY-@=n;bAVEW4w$7*bvn?3m3XJhQ+VlF$44RVN3`5!1*DeGUX4pQf((~PR~trfn= z-S{0F{(bJj?$Z}*9jrt)ofRK!%J|Xa_R?dmZ=kKJY>Sw|HAF_+05?_?An#~;IaiFc z6oL8b{HGPlPA?|wmf%gs2;rNoN4}%4OG6WgJJxpapD0EMecFh+_p(2P=7ah}mNwDJ zf5a>XuR#l_oy6eARK=(4>Md0W#ZK z`!GeFPfxm!c+_mgM#*wl6ApG9cj`ZCfvv&aJjOlJUb0yDa0Czb10v8y?qRoUO0^8S zh@Hg_Lolg0V4+D1mN5P$IIH5z?E%85hT#VhMNaF3*oFfw2hvdfnzn2AU+Q_Zl9W#D zOXyic6HMdG?Vi#as=U7<*A}ky4!>aG1Epk(Kb~tB0kk1sJH+<{dJ)q(wud6E#l|}0 zncH0-{hhL@H`+e*zFoajaA^xSV{*2j5yCQ;nuU?siljg;s54lReSMjn-4JAO8RD= zY1aK-J_(_}X_|?b@!5;a?Fhhflz*TqVys_fpJlPK=Kg z=yZk@(6*FIe&nO*uz^26VAvpa`up4m5q(54aNcVIRo4Hs%hrGzK}wn665ymgPn*Px zEA$@mR12?Z?Nlr$y#TUE)IBu!0v*eHsMi13|N2b}LYLjymk%<=x7VED^oavp&Ejr< zc#8~RfE6ecW}X~zj6P7NSv6>`@GP8g^b-*fd+9qr!}!qL-N&y>aWO6QQM(V0J~K4q zPxWYG)a69&WUL#g3sy#1dAKNCfatU52RPf}Au%Zhd)jgrri&8Jo(y19M6#vDnzcjqr5<0OeSskdRv1=#0YiixX`D1^jjBN+D+uR|? zwC1JlFxH|nDfn}AFjNq{D;ACzDRj7@GgOcxuaRQRCa7M=tE;f_BRFk|ib;cP)o3?L z1moUE#(tQLo!|$C@iZ=fTbn;EC`2yh6d#N0k8Z65M#ftAL*XE&PBPX|_yX)IWIh`W+n-aEEa5*ArXwPO{!nZrVR z^Ur4B4KB(#B5bE3r_SKPa|3xDO&K!FS!zq)@$elP+=^-L<`_(%7^H!cDOJX?*_q{t;?fA5y?1nW{WS?d=BK&-; z@B=dc=%EN{Mesbzxt%rXlIJ`UEpy8pYdx`O$ipqd`1uX=_AzO6b(yD;t5IusKjZk$P2PC*9%k14D(q97G(+2zn!1H?Dvt=SQcATim%^E=yj8!j4-ifu(+taY zyd1N+qYLmOWA9*2|HugMU9%ESZ-vUz#;Y%T7M7ew3F59xY(LMdE0wmdZwkq`wLUJH zK~;7W_NE46S&(`Db<>tWL$}`pg!Ky`efr-uNvR|hT>rUPb7r`_>Qx-iMblZL5V#tf z`rX97((aL+|6;PC?=i|dsRq-WU95~i62IZr>o86;&x$ODnoR)&fg^Xwml;N+22Z%- z(SEwIB6b}+8u2>Ct6648rYAV!x*T zK$WZUNQX6^x49@@SDV41)pFthcNSlWcG&8~k!vAUS zly=}Gz<^Gf!;XM$h=4X$f5HPwy3hfm&_>A_ayptOMg$%6V?~ia9dmM^zrI$uWa?wv zYTx_mYqkv+nBXLvD@%39iaOj{AdO!P!Gj8uhrm2b zh+yd*H)(h|4=G%ufnD``o>IPL2a`$eGHjtyetgU9M7H2hYe)R))D2#E5UWky@CflA zG2U=z*Hq%+8;He{wW%O!5Q||Iv`VOb%mDtZNf;Wnz~$cigTO*#xdVCLo8Zcg!OM`j zY6!txq3n&edzL=U&@1=w=vdXtHywE1pmGPW^ z&pFNxXbD`+39Z}S?!~0NE{#+lWMgF#pRL{WooTWLw*=UnbBKXh7Pg=A_`8b9@xxNz z$hU8dO?uinReHzd^XwB4+M>XgS0m&WKcH9+E>aebh0IE)kK3^0Eh=<#ZU|7W+rB~1 z9Od+>iww~WIi$^u$yZ9XN4wfF->jHrOwN;9?pr4-y_`1~!nfn3#x;sqPL8(l>vbhu zEmJa6!M_jsen~y)&Ri}y!V5YjZL{RyYQ7>@q#-?CC=;lg)(YD3z6Kn0z#VclI9<{ca6h#=n(`PUFR8vtJac%8M;tMKk&rcFaOhdg z-{i-&@NI3aC+Pp->n*_I=-IYmoEEwiiWa9pDems>?#tp7TYPbMD^e)#UYz2K6?b-tuq&;2vNv}AlpWozo z2dPT_`es-zd?`P!3Kb7q#3vlM?;OzR`Khk*XaBZ5iCh)Zq#iE}rJSgk4C8iM-e)wSAUetn27?^yerXh}oWJCx4n3AA|62nb5s({Hqhd%x~~w zqFQCV^Zhxa_49{B)Lw9b10!g3-sm1@c8gei^0UpGQm4et%}u2i1oe z@Z&w5-@F3cMaDT`eqz1N?i%v1TTiwQ3kXKX8Ji)P zVGV^uq}2hjq^FD0kCo?#<@Y!+X4Eb9)Pehg@J>lz#=f zGX2^vbTaJGHIw9n*B@OoW(Q+E%A(xQi0p##@z85HJVsvzYov}wUCAHh=+uf=jBs~X zm+UNDB=f&-9%<<*#Vx(&B&82*{QY6RImS1J!GVOf;XAM>@f4;rDYeF$h@Te%*AwCJ zeX0mVMUyS+U#heJF^ui|Fz7=*Or2S?>hl^>qA-IeYnD1KcX{-c(7;e;aWJvMOmY;w zqBd*lo_aS@pjAo-Ec7Z!!RN^eKH?6RCpKrWOT5l7(H0i5>^ioJ#&PZYWzHrqc47n2 z@C`SVNd^{KM)cI7x)l8;w}n{gSRxccc4iJaA1QhR{qExc%*zhl>YO)V>3wf;4vuo-TFLu zML20AUR&RCDW3~W8EoV34q0)%#QH_yhYv+QcxikjZ z;)Y!++rmMhhk5GW(OL_(*%-T0CMJW!&7ITAe;qo5LsR0-SNkCL1*kp?`qu3e4Iqqowzu#6|t}n7$O=7;f3=X zhoyW+h@SktdYGt&9l?=QT!{BlqFwR_op~4Q;^o^=_oRo}${)S55NVg>7+wEE8onsT z{Qc3qjDtA$hGc^hN){a0N0?jRWu_xDq8u;vjhehm&B-vKvnHE3ExxnKX}>my&25A= zICLqi-s+}*W1Bf6J;KmxMOS`BTpG(8rw>Y4!C^cz>Cu(08ZU2}LV9ZFEMH*irBPZXy<_FY%;uiFI6$`6Vd+(^Y#|?w>Fi#-p(BoSc2a!B zTOZTOy$>}`l4WTM>*eaFwC6e@E?wM%h>)|iz-5|oXA?yqLrUMCr|vX=LR|4*PCaJ& zXzD&IwI_HnNQOMZQ<5<+qc8$JOe-#L%yyzNcpa4hMd3ToJ6~#%bv@ytH!G}4P@=ni z{$?}xacSGpZRyoV7&X}$lcTG%E9DQ| zjo7CULT)0&i%66;%8as$G`=h8-AtUo3MZB&8l!l8f}6~=9agulzs8ir(){OyH7?K^ z?GIe|CDZLEel~K-0E2i;5BvSbU_k6zu->u(N z*1q95opz;)xPR*pnI|asjkW5^V2#gt* zK-e~X5M$&b3<@K}it@oPb+!6N_Cmlc9pr7<$)$Wg>Cr^;Nh@%7#WiIEI6Z*e+gR;D zv=EXf6O@IF>?E;`WQb!N?*``@n!qsqo7@{WRrNdWvk=SjZ0TR&N5N;dM|$OTtt{p& zPBRNyv?W;Y2XcwArQM*#tk952oz$%KvAdmq*+bl&s9>5mLjU(N zAiVB^s`n1eyV5I2yF|up#Mur;?WttW*xe@~rvlgx)>VIFP8fd-tA^&_7>>j?qg^*l z)BB!u;9tEHF2Hn7x}ua$&qpXu`c2{3f)Z{N;5J3woi2st?0SL?D;rw>0p)36xz;GQ zN1f3>d5x0PN{hziaQ@Bfb@b<&qp;WW@U4HZiD}e8LhU{r@b{i+*t=hU%^>@4VJbq- z1pZr1{IF`8F)l{Nzk=o5##lGB*5ZKtJP><;L_*yYvk zYXw_L@eKhZQ(^U*#~4NTdhq9Ufo*O*%JayH;ly3KEBm=8--)a5KzOzm`owR+exIOYKN=z~8AX-P`H&9=kzLXt~UF*%F z)5?!_hcuFwbK}U0fc(mooKd71vJl-ZGbgTiZ5RoKk1z%Cr7OxcajlFhgP!bGb7jJO7qJsIK_o_|`aa80EdUUz&qx0=_E*AKl634!=OssDXPJ)d-oGKthuNepo118(tYVSnz>U{}5f>&n9W4=PY-fJ@k9~sYN zOQYiRfW5Fm5dQr;1Bv9yKxLL)F+s}uFD?z6>D13`S|$K*iy8f<&~^S_LeaTnQ)Y@> zxIN-$!Zq9HPvggBIYFy_pa1c1Qru-PI%2{Cq2WGzx1cOCuiwcuVXJ@O%>2UsI-ddL z!C!(-{yB5)IGoG!6+cJ&B5z)&85A<#eXHOWuvpa?=$z^+TXina3vuF8Xr~rDN3841 z@qnLieM#=+duItjEp(^fwfjOpyHLm2!`gKA-U_2Vzf6h;@kaGU<%EQJ8m&lP4!euE z->Lg$qvbKnc1P)Eb$sRR1~n+spm(6~80{xAv&r`WS*4sN;^BCOCaU4Mk$;X-kdw5X zLk4z1F`vCUK9*>EBUt0Qbx`Nk)^-*|j`4bZCds2J3Uv_wpTh1%XE@9Mg3%0&+*IGSZ#nnQT{*3}IN{H@{rJ zR}F9e=yr*YT47$wS_Ev47W*YE{c??Kx?A$><4+>H1FY@;x)Yn@Tpk1*wPiO45u!a~ z0T>HKzVq>-5jELr$IN+q2XV_r_tV&F+q>uKfz+vfx`vojzO1hqZ?7piK9wexa^L!p z{901xXNUwa)fn==u}&jdzXn&SXupd}K2DZGUZ{HX8}*xh0l#El8D6i!RyZwsp8w6g z8~gEjhTEGo8JS)awwl>P)ua8W)=kV=L%69~<;bjbV46$GbwYa)XORI+bqkXprFt!0 z9Q#wq2c7FqTjVtNajvu^y}d*9 z6TK5FOG0$8H#`Q;@x>$~2?q`{h&9_JmQ*qklr#7hGs$=j$CF-eTeLR4KVy>e8jj2Q z>q#Rr|B3SJUr*UO&!eb%2g?y!7IIiY&f8kc^a3Kcn@b5bXh`hWzwI8R3(4AD*LzGm z31_WL_ZR6NYr9nMTcJg2ZL8}58;NzvUTeBW%*;71YF03ewsCe5{ZM5Sd$8s(Iqe2b z5AS?6VYUdk|HS;hcsE+T7SIw_BDxRgUqxz2@&EEgwB|t4a>d{+Td-i|Gzng8DYOC; zyxgD0ks(<6KLQxt0;5SS@9sgKH{07y5|G`=Vr1;Z{pr{(-%;8Kn!VOu0EoNNu(cI! zqs{5}$wE(MZrr@yb~>?4&erNtzab4g*K zva9anFN=3RzrMtNS;=>re%)+$VZkxQ_!XRZQ`1rV}FZf)LyVJRwA-?o7A z$XsJDFrVrHiyH{*Pn<*KcYZ5Q;eu;-l6{NjfcNRyE1W;^&0CK{Mqq1uYb}+ZE(QrV z*BO$x$XQF*4*AJ%mfBR$TragQNj?UQ%`$rBu&IA!UHFrb5n0H9X{LT8z2KguGZ9HzBk%avd;Wn~ zJ!-R%i-^w?tL5?f*1daLo{0L9z3WcNsXU|L+0`&Z=teyXH$}Z=u@%r@1x+_ENvFV- zvDwN6i<<~L%1+>VHSx7GRy>fE5&c{5^8e4Q!2JIhP#V)=)(~xdu<6}@}W;j|rqJ7NDc9K^sN|<&PWFN-B#Qic(NuxGbQ|FR6h-wYDNE9o#4SUn# z+KQ(Jmie?PNy_44W(WGMw@vGvSeN``Mo5?TH2?K~yseZPaw&Bu^3njLjZvC^H8q4i zd6)tdp`f|{t3f|NUtK9-UE&0sv9vcHLnptn#HOFWOJZ$I7P`9((7XIXYxQ0}dWoyxXK8`gxhwTqrB)JBzcvPJF$ zw_CxVC#SDxvp)^N&*iUR^LIS!FUG*ilmnutlIF?HafgzW&|3E&yJ&3qVJ>{=o?tJt zfC4esDx!%ymGdMbchq*mBhcIAP=Rk^_`fg)!r(Z7{zovdhht##OyR=}Z=uy^rb8{Gt)k)Pmr_rffPuARisGu1W z;rO~5JoSRh9WV4@LVzx&xSu6!H)%gVLEI^I#ysoz^?)<2o;$zDBkvugtGM+K+%LYm z^)0COWHkqx!Y426l%w5UW=30azR8{$8OlPg3L<8WxdTd6^>iQl%_im7`DMq-A5KGg z^{{MNIhTTCpeGz|3$&emky_T!jz`{&HeAWw3ahf2rw~)29(rxZ@k&#eK{xn9G#W94Dyrm4!kfWOnB+nTwTLMV$T@ zoVMHjN7|&l<}S2bs4$+u+EAgby40x_G%Vo*LV@GSkYVAA;0JGh)zGkBBzeYqxRuRN z>!*|ECPz{CVMikOLQ(sEVh&NKwP$0P5hdI5yFdmoCg7tGvNYMhkTVa*QSrHcY;1?(|@TaSU$lnQ?!Y#_u?*$=5RIUleGkV zg8TPjmaB1DNZh9f*sJ;2vZ<*cHGwD56_qraYUwmJW6E67^YVZF%&b9+=B4jK7cloI zK^}@=F9DLT=*>qfSrihb7(_O`Fm~N-mv4OIdvDWX1!GX&%;M;H?IP7a$Th9*UpSho zR(b^@MnzT8@Nz4X-o_bmU35+WI<0tLb+ukA6Y_;cs$_(Tu}-XfyefqpN%Xs;1}=LX zwf=({a&Z%iy@5TiZzUQjaGHeg(BSr6YP4lT%Z@eIuz{KuXx(Bzdwn9Ga5y~a>v^lH z4NWTN##O5g+ALbe{M#)0sZp|}Au>F|z`q|wb+z##o1LQX6mymP*-{)qv2P8aUf5wl zLI-V*_3NgW;;5l(jAX zg{V2)WFC#ZtoD4iIg15IvL44xUkj0WXB_!x41A;JO|8=KDK6~0z7rz8*O=qkXg)AhWr2Odnuci3+=pSozVJ&{A=9YWA$)+v( zfk}_q)K|_ExJ}A&Oh!HY`Lq!ho;mlN%=oQ&Eq*?6$~`j&0{fd~(*lyckE`c^cUtr! zx&$>C^udxO!;*4>=V5iDaDVDUa`ZFzL&S9}QOc{8=+V|`e_g)cO8jh^Zyb7Csle`6!F2Ph zEh0X}>)`SNcO)Jjtf{nB<7U3Gqi!Hjgrl3SYMGBkPgP=GpR>POca-VA^QMend>G>! z>x}y12ISBevx*3GGS1(EEz+p0{kc>~rtTLF!p`A@cOu}YgWc+SR0}59wjoOL^wk56 zLF0%Bw^Gi-2LT~l5R2u*hdmz(9{728%IzRB`)s}Q*!Z@MZ~esPr{dISbpub>N^_=$$o#9^0Ck-%opl8N_rTb7WaRkT>=?M^kX+Nu3r*L z^&Aj6rL}?MM<2ZBsnahXzO(2SmNgw*t<<4J#&-wWYtIng4`A34loibO%$(6SQ6w1D z0`=Xo(m^GzrZA*MM|yMg+$hkvG#xkZ1F82eTCit=%ab?-N$2q zm@DvOZmW{C?e7`t)*EjyxkKq64N11+0=C3O9g9ZC^V{6I2Qr^CW%IkqD(*Gc-uO8) zfsV^MHP7ec zWKU(Yk^tbwOjTYgZL0(ffhJ@zFFWHZn?!*F)EOv9Kl(&qamPz zM6^0%$91x$Rv`p^<_>ZV8yx0j2H9E$>^&y&r`Ro>sC^kXOXJcWg@?^I!J8JqU7!RU zEU6w!PIgn)#q~Rc!7cMndM86|A!e2Y&h+%z<`fqLTU3ouIhxty=;}C+m?BO!&4T-W zEOm8B#1qzo5BSfzeyAzM^rIImb)I{SR5l;3naY|~Av&8Kk{b%W>f2z@(vY)4%a7?= zs$Idl59}b;kGC5VHMIJ^A5m65oJjMvQNLiz8+7^;ub_?WEoK*(RkFz`zZQEO82I%b z`ItjQh-MhGj(6;{qx*seihIUByvL;B`qd}!^wa5*Amkf0SgB5_J>f2q?@sFeyM zOl;|rMl1{ba4?Ij9pvCR2rxjm#ul;a<L_|4gy`yj1tm?c{MTqswk3p(k?_Z0?M6%t-LHi=H&Oi# zaMF`@`L6+R25L{!RjcUIAWneNCYP1WRFvk)y0Ue4AZ)X^y-r|hM<=g;Op78FZ{-Hj z@~=UrX=P`F!^Ng+w~BiYOl~Xvt)=Cm?eDEk6lchTZ`~Pe!dmeupS!lR|H_G-P6B`6 zFg4Ew+A%>7UF_|x-4fj5qh6>q zC4#SUM>MHyFUIilI|bG>^IcX(a1Q{&uT~qY24z|CgOFh@;Bs~?`rk^ct!v9!CeY-q zaG-9AvF+N!gd0gz4d-9{4QXUweu?dODC1#*TP9M4??~F4Y*MpCDDJVhC2jgOzW?l& zl&vuyvp2a~_p&P4CR2LA>{1$qR#4u`+co1(lYThsm{-8iHeIcGTGrYk$ar#6rX?5K z8#y2PGXgkkuLG>%c=E1fw!AHcA*F*ErQqjUO$YaN0Oyxl*&;DsUC9Qp4{aRr&#al4cXZQy$ zGSrRl_xXO!)$>#ZpPne>nXrx$6^E9y52Yp=4nmt=z=R-$z75ORAo0Yb)p8xI+9F)( z)W&`pYvFEF4J?4!C+%eK7VSX zqZc2&ZGQIfShcQuU4fsuUf(Q|ADK8gL_i)%gR(MKI!PM;d%o5np&6}w41+ry3Da6I zy&)E~J54hvJ$1Di zt{pQEqbzZ?gP^g-%CUjT=$8=#lMVK2$C*RxKDRuxxNW<2d}IeLdFrHH*`IQ<)VR@k z1u$UgN}9Pt8jG0QAVr;E^4aQAZYwkT9q4ekoF$#u=JPQnMrSWN4vWM*YhLbI&LMhd zo|&h}udyj1f9Fr;wb2BS)8?38BWdPFX6YiFt0eo(b`&UeDo8lD?i)7Zrdd1`uL4>j zkfyPl6*-vC3tZ;rICDO5G_j_1S|kaH{x9 zye*dS(S&AP^7{)_tBgpH9ivYq?*fWCG&&CBFP}ByR!Zyub0d}5UfQqQiirvkRtQd3 z6-cP6S0Y+b)ia&Sv|AMu201D%P@ayIKMa+_sMTRGiQcauxEjh5>$W4*7rD?glPf$0 z9gr@2y~h_WCf58HdLq$VBzpc?pnUWE?ed_5+VW`-y*U@JTz6ZJsoUer7H& zvLDeOx_;PVT_ZCONA9AiYRhmZFdtVrv$!!SXNS^Mi#A`#1)mqAa`c!NtvJ0hW!sW; zekv4?2O9Vo|4C-`Ed>mZ1xi-`RnC;Tbb5$jHDlkzkKrnx{5k8Q?snQMf1)25oU zOgXe=1Z4>wmOIvm<=7Fd#c4pv4$vw;2H1te5|!gfG|N5ux?s~AYag6$CxU|z% z%9PX+O;ZLr?mQ1v-PuE4Forwjx}Mo#EZ}{b!6(+TK~=#CJW<+wL$6w7p~#`VFZ|(- zZIUME?q||0OdD#qXj0uM$>QDICC4gUuM~=04edC@^1=r2i9=~?S!QE!kzbqkhB0<@ zZFnrcrT*KOe&8lelw!0(>-p9s+9Xn=Yd33=w1Il1(=(+CG5vdMXw_-FrtMXU#iFf* zCqVusIa?5+oi+ryQeLii$NYyQX%9O!y8ncob%s`pPs@mwJm!yMT{`n(Q0+>$gUbWG za*nzSQ8L>)_g^&@6mSeexn)B1!iR7B;=kJ4VQ=z35lRswIIaZrU!@4JVE<|u1)XZ| z8@>Pww6%&oWGe8SkS2D#XfVv~zt=Y4@fKkFmVT5q4lRGuTfCms6a60Jm9g4u#9(Uh z#FC%xnf|HhTPI~*gSr>KN?n;iK(v`-kB-QKcVcPBDrx+&zVaWqsZ2AYI?M87OG2{c zG9)#bdqc{M`Kzzx(bJ>7EkrAFQ@iMzI+p|2&ds!Ac?lU7F$Yl(F?)xK;4Cw8ldVQS z>byHZZr$OfJAHDU58dh8qS?-(J+57JVBH?br1ywZ4Frd{)h!oy zLUKHkMKoJ$S2!-AA^tiHVnbh$HPiNry6RVs(UOyp=!9`g@i0i_Q70-nm{{VWK3}3h z)O`%HZq*i&AzISa9xGo^5N`0-#OXw4Nvh?`3Oq?c%<)qLqtyf35zmp`q za4k=JJL&zk5yG_(-!mX|_=54&J<=)@xGNlwVTu%dVpJ87R}21oid*J-`Y#EUN;`u| zcn;97&>uBuyXc% z@Ze2)lTN)dyu`K^pmVNF0vI{as|D!d+iIVZTPCDJ80|tz={;N57(Na@97Eu!IaFSE zRhgM0em$hO-qH3Lr00c?x7zzxn@R|U|I57)KRlqdUpn|78iqiDZz)a=d6=wsbz-OP z52h`F{Uke3CBlW{MXdVwd#0ww>lndya{cOjxIR@@mP=@)oN0@uX?(ndj}TnPY1YbJ zCKY%7O-rV^DdK)Him4xXdoi+eUp7V%jYnVW!oUcdxCf*Jyn_aB=IbXc&(Ij(Ck8yv zs&pLdQxtiV){#CC&Se&qjXE&yy$TR8KlL0k1rXe7a6OC{qAq2>_nqnvyl|?%-i5Is z8?|KdlJ9<9ca1Hc`98L3ML%f7PgFawvkj$iv<2yD|K}$Ey<_+?e|#jSSGo+Xq4FsH zH6TnV1?+xfuM7c}<#w|vZ(;f<;%OBpA&pZo19!VDK{cz;@A1~UuhyGHl+tPCPQB}K zp?mqh%xEju5TV6x$rD2<*zApniu^b->8aD;RlrdDlcW`lZ_Ji&mWgk6;b#QfTr-G% zRMX04I1mWy)l>1k>Fhm!uD@s zMZ0eyQyE6#(q4Szi<>t7kevG*Q(|Pgf0E<8=TP*L(p*R-{>nLO4%n9>*aVyMHfCB; z1eCk+C?$wY+^x%3} zlr@WQ_nFl^Pgic0iXW!-eJcIZ`f-YW^h4{%)ZcG?RQ-a1MEcP_2ydsy9&I3pxW5IC z_f7Y)4L9(@)J?MkG1wJ`eEL-pH@Gr{XB5k8`jw%FOq$kr5fxK1L$;>HO`2t^6nODc z(_7&K$M!BpjG}&_TmyE6(FD`|AXw3@#y{XD z=^WgVNdCPB$^q49p{@U;7~pD9>FgO^kStB1mtOtfB1w@@h5lcPL8GQ?R<-7$`f<{# z-#iXqYTA8QzSKA0`aILMGf2ltZ7DyXf@8o~lcd=eJJ*GtfWTj!!IdbQI(hl-SZkQG zfL$38y*oWl>K99F=fX75d^6KBT{0&=EuS^=@gwFO=6xTP32pYaeDPg_iDDtw%P^0RWK%@)bYQ1`zi!URWQkY$H+HcRHK`6>CR`@ zOe5-Ti{MhDrfO{KZJj|^q`k#yxt0vWS)#Q#jI5x^x@TljTU*8?a zw4LKzOfZA)u8wfdAgkE?f!od^GER}C-9U^}@3qgYoC4?t1?JZP8L{r!h4$xY8(Q8( zn*}tY8^)o{a^`9MoVGN!dyl-%1|ceQ<;WbM(sB5<(TX=NS)107e3+yPNT#q3e1k%skuYVq`N682g$L-Y%M zHej6~EMXXX9RnteuN_9_iUQ+e|NDc3vDaZyu!>^BIP1s*;&RU~SgI#GH$+s7zW&Qi z2QLhprFc(k!Rs{Uw3D$j%sZR}Ns{M?~{)WvaT#lKaPw`sb=E z_rZQPYA$K(DX{x^C6mr}CFa1!+MYDa$WDZ?JF&Me^YOfEWjL!-FB9>a^=r@ zzJual#x2(O8gQcOr(L{7kRk`-S5Jf4PVi#&xeqvAhLuplLU_Z$v)voGBllC>Rr$ zOSaM~nK{_Tmp#M{3P2(ae;f8K`vtkeoqCTHApCahDpMI#srG=!^a!@o%sC(;#{{hM zE|Q1bd+aeB-nGD1+myS5GxmrX@ue28;^p$g@3=`OF!p?1EZMi61FR%bB37qr(8o;$ zq2pa=(#lN{P@e<2h8^uYyV%yx8DDTy`n3r-LRm_|A74ANZkz8yoGeQmyQ(5_3EU9_ zS$T9zt+pIy&hBA=4R?L7GGa_?@-r5B8n1A-91;OBtCa6Lm+lX>!%+A3hSH)E;+v2F4}h(c#OcAE8+rZ#%QH&xFi zpILNu%t)i&x`9T=4ZF_;-Pv^mZ81{AWTOdCm5>m<+vp&QwAs0i8=nzC8sm6f~>?KOBf{`eCH^ZNpfo%d{F0)?BuFPkPAOAVcf z^?8mbO+Zlbm#uB__oU3X99z1y{ho_dY&Z&EVfujUCQ|YS=WDwX* zeXkl48fPB}E$wTvXHdDF?fkw`gk!69WwVUd#Fd7njl06xT}fY~4zEJ^dgfe*E0XdE zl5ow0qw#%G2$*KT_-gkL+{iaLWJ=%f6)$7~NAKe;l;Es75P%bgRuG_P&#R_@MTYQm zViqPOYc|}ASpP8Wk}u3@)Whos00DP-!hD2RnZsGC-p@lHQ=XPo3AWYrQ%~5>P;@E$ zcQStf3KgMN$e)q+$D;P-E`6wZ+o&^EzcIeV@z+{0zVY69Hwb#2zji^Nk1q--z{hsh zL6~QoU|;|41%;=}b!OdrC&NnGbL1O%LeydCJGG?!d3?Ix0Hn7b3B>=dtDm$_G~o|) zfq3P0gk)tvtu96aF@%pobdbpR%IM?hcEt0;q%o3682Fe_-lL$lhQc$?IKGof5c`Od zdL__)SyB$L0ld7c`c{4tAEbnd?z-e<&i!^b6)e+I@q^h_#u81iK0+C(klj*l83{qX zN7I}*__#kwP}22YNe!ym8-KaH-ZU(Kj^ZTF zk6q8tUe8ZVC#ST}H3T`Qb_8LNKW0bJd;&<>nCqVtRL3f^oZc$g=Vgm!%Vn&I3We|r zMS|yD#U0}PQTYX!rJ@^$*qvriV&>mo0FNO*@UY1L1#BNH{8mp16#w5=ehr*b*Mak zE;e$}W))3FzaZfXgEC)Q^eYurncHG5T{_OML~(}*1%=yL#5|I~&otA%C`MQg3IGk4 zb5_K9yE;<>bIvr?w(n2(a?*V}AHPZzVonVp_Z9XS#1RGWx6vq~N=h^jb14NnW7o6v z%koJa&`9$Mqklx5SFVis1NRzKP+d(+??xY3%3`&3pPi~{LiA}dxKvwUbKL*)@P;_# z<9Zi}3?7-ufak1>hxhB*eWar$qq$A-IQyJ)Hb!m1y-Oxdclm)3iG}R>5ub9H?vyv* z*td=UMute0nLu!AHe6_j+P|ssIanj8oupWfbjZgyOaoKQOABaM9j4EZM}9P?E^8wE z@+|twn1pk9P!7c&402s{5S#nTK__^xlVe+XqAQAPHFwQ9Tt$|?8&!|%M6_!70pW1)@zn=b*=o`ACxI#Q20i3rfR@3m{^#M(z5I1uT%*(3 zGL_Pldd~h_Y&s`hb{Nx=w)c*w%f%T9mc^$ZLMw&ghLf2LbvS-u6bl*+A9bvYCQ0J) zT?nAQB6NSs@LiYsx(jfZ=fl><9QmG%sViH;9gG^#fiI>erXA!_(J9KH$OIehp2kRY z?(^?e8te72ka?>}T{yY-M)axp2a$Q=bv2t{T4yimAxZ@*6^@(gGA0CQ#EZD9F^iT% zf?@IQR*@XhIjgpbQfzHx>=EHh*_t*9;gEwsAkyM4+@R7*Q7W?X53?DMKKJ+5^cJDo zK8>NzEs)R;3qgyB>^04<8{ap|YeF`B%nI5l4}^8nI;PaVCT1|Mc!Q@i`u9yt;u8Y3 z2@}oZE?Zlro`?sp4P<@uk>~Ez>lV|R*;m1S{`$H@m0h%A0zcdzh~z`OS{OM?DXY1h z$Wh$^jpK;gyNarfXXk0`HqvC1Yn>eKrV*Uy#Bcj1JBpK8x|K=fn=%NB>iUr5`uLz8 zybe+gC)Sw@Uf<7@%e?{U`IBTBgmw+DP3^ZFIv>6kz5S@UcTBe{0Iu5QWHTc^bn|B+sI2IggJ4GDHR@;oyPXKOA%_JSysL)Bx zxgf#MH=}u_4Q=e4tJO<|ZSOAkZ4pju?oz%PAc0aNKJ(>z-XxjweTge?2t&4VH3Y7A zqFes%2ttv^^_P~~+4^ZzZgA7iG%*~AgW8>-tq3mP`jA2~!5e?taNb8En1R%FC0RxH!ZZ^fckwG3l?HEmRi2(KPmScozsQU;1@( zFKXw!#aWc~f)Wx=?YzhfNkXYtUM#u7Fgc+XC^Bf47A0yW>Eht+@=-dsK7C zP{>_!+uZvd^Dn!I?=Kd6u#80Z1<-a|y0$h1B#zKNCgdiaY^5Sq8%%8w7~0xpkqb~1 zTcanhpO!quYRG{Zf0D2rk+@ULe*5v{|EWc3bltf<=@)_d0hmJ;$iq~dOOG(S%b`(0 ztry=Ak(?4!Bzx&YS^0D=M$RpGz*l9cMxbA|F@@Ocy<#IRDI0tWdJ9KylTAJ5JuVDi zZmp~K*kQQI@eW-=`Uds#P+EzEgB$23-8gVp3I~U18f4vxD!a*-)9I)=-o_A3@(~l1 zhs4*q4vG7Ya43}ZRi=(M2n>>C~utkV_CWgRe*4gVdN0Ot;CosFhXnoU+D%1Ggv4*6UyVl)2dxb#z=SXKn= zM_=WN+E`-sb+xvFO^J9_tM6^k8o2z2qOiJj%ff_7_)oCvv42+R4*D0=Jix&j49e}{ zn6u;F<~G(9!dZVH{mp1zr%gA8{Dj$8EmM(Nmf;rgR>$vFNq>$e z*kn2_4&B8p?W?@b=haP zG5M5&!Ll29@X?6z=84@MkR`uLLms>QpkyOKUaF|86Omi9N!vxD?)ot^;K>Nh?f1oP zA)Pvkc~YJc8Co`WS;Th$BQf=!8hr-GGm(bO8qYM-084KtU{YYbqy`!CsX}*W+{9e5 zm0t7>M9VYVVK4raUiC{w3PdMDG;6gPd*@L^_Px@K#E(Tct*>sP+cy#FU$TmX+|8R@&&FRo#WfpPM5dH%>98&Q#1!r zK;+hC5cKTHx7AbVKC7-fT3`H>z>HlrWD~-hm0Z_~6ZCpJX3QG%nhSsd znEBo0PR#`;mlrcO<+QFr$nOt8^f^?GcO2!F)}_Fx9I3^L`)Hd_td8zJim7gc23tVy$p|%1l!RJ5$A*uLCY>1V&v6bkQf-rkEDl^Vqq|2 zJ4DQH0a4ZCUFSoA_fZ0Gw1?$&Y0wMSSdnkexXJHUz9x^&Yj%>yv3`3tbCNEOAV4VD zlr)7MD|wWdEALi@)>l|HBO2hIbF3BNlyf|@3kZG*KW7-Vyw5bh8#qJ!-gCq_`vTgy^jj`0_HGOt_j3RUk?CFv-+K4|_P4rx>~n!^ zM*M0IdS$ZyW!0#ih!oAno)!dC+VJ8PQ6lLUY@$>Z*n6AKUin8OW4771g6<4(sB<4# zEB$aPKSd+g(q=Zb;x>xfY4Cu)Y@<}x?EJzE#p5boxcmdhR@_yq(c2L1RK9$>HXk+1 znJTiQwnwA2Tx4CSMWR!UuEE;P^Ax;D9F?2qk-0uHDU}khv&2G}sbio;i#`9CTubg^ zZkbIzeqQ&G)k;cT8`RM|ami#dN5-bsc4PQ}2u}!)3P7+)ow;VdbjJPU3$?7z9W}gk z4(t$Ct6^}#BluBn8KrCo@62-iTmEA2?E~u^qWyzpkD@ep9UxlNVzm{eRvxS8&2nEB zO4!ExDBmvu?fC5j52^XEH&B&eh%prK;V5+BM*N9*wp(+*8$E`eRGCL&ib6+({M8bP zXYY~qXQl!0=cw8jgWMfjPCMjH;Nn|p5BWE8Ode9XJ1XxP7`~HBzA~Aeq3ONcG>?b^ zDdc$)b%0-^h*xLRT*LOr*IaeKpV45gZ5}<-bVB0hHm&XkE%G*R->Vghnu=ioCo~^9 zY4dBiT+(`XY>R(*%8T!j{rKT)Jxh+}5ZUDlCLHKrV@yH5CrcR*FV+Vd|r)njI zftIDEuA0U7;MQTUk*La}p)0PEC_)Q=N2wXr_Pq`9P0xFX&8h$y%^(3o1((4DC)(VL z+6-&*BI*+%U?-Z7^(s(NPdT3{x88hJh7PexQ+LT{YKKKDK4<2zCvoFS=2yZx9nc~z zjoo-hpSjgmwo&t1fx7ujosElQ^!|*k2S*xdI3P9WnUdWXJ{Mrxc{cUa8ozR)6D?%B z^Mrme(CS+TR-i>**mI-?XNbAcZK;`J#^(^ro;NOT&O_$4`jA&>xb3UjTF46qAExL~ z&H2SeLmkq`b;I}dGBqhymhsdquU0h^oCg}PE}o%Ttacw|kAwhr<2a}GeEug`r6@f1 z?>QM2Lq^OLrHbnNJ_~!R1&{#N?QtuCwfZ|~4J3W4=2e)b!67;!K+F5rA9CuGw7p|R z_TWSVm)rnQ+dI9a+k`J(ze{)Edo)EC5{&i)tnbbj^63y#uFSh`FJjrve^jEU%=suhHr>D$k{VKE-)?%r z&1RqP5#u|55RyIqkrlts|MguFJv)kvJiN8GrCb9Q(Py?_&;ZW|4~0@hDXBXq@}owN zC*dZ&pIQe&o7snP74H)#opOKbrN=m1lc@9GaW;#52x&Hdc3U@2+C(-^;1EXoHFOR= zw{;*KtLxSINhG~NR&p78iKE?d!uI@DHt7Y`s`i{;InM8E(NAqF$YmQ%NBaO--))jG z`jqZfTT$}ZWj4kiIO-v$NWgt7J0v~i$N=_4h;L5*z*Umy_$z2}Q>QmP+)! zf~&M4d(1kFFQm~N5zWB~Rj-u~17Dm6^j4*<$u*?=l@S*(@V?6=r&Ml*A z3n)%K$jPpOhw*I*HXA~aTkT4bAO<%izI$ecaag5R!)ob)Ww8!FaKWr5L>kh=C}^%e zO)S&Sf$WlP*k#divr%WBUWHvnO6RF zbjunBZcT^-;YsSo+wPO{u$nvTGtJJ>BSPrv>{`I|1DNy>;}c__SWj=Gn2hGp8O$tJ>S zyL?vf<3(-%zMj+a0aHIUx=q>Cn{R1GRwoQiW>fQ6AL_I;-RW^(N2{af_fUZT_eK>a8@Sp0 z2ChPx_9eZ#P5b|Nd&{V}nqX~oaCZ$BB)Gc`1b24}8r&Tc+&#Fv2X`IZb#Q_PcOOU~ zNbcl4-?_iOpLg9=dwTYs>00f*ySu8MA}_n?3?Pk18}G2ULam9QOKs({m+LzipJY$l zJEV05SlzlX*?r1P>PK!1?fiZ5rttclr)_R1=SdZ}v`*-AW+M*Qy!B7iSrooOZ_+O**CXcz4Rz&EgjVv=V7GDHOS*=vKfi;7tgcx4b2AQVhP6WdO84U5(+iOV zJo2zcCi$!m91k9j^*fGj1q?H~`C2bnFaAfX*8f9`R`7PN<<9zX#kZvlH48Nt>PTo+ z`ye~>KPA1*M!-5m=QFCA)o`hulTdP>YXYqil36C(M;W7zN0G-}S)L3VEDNhcP3~Mc zLaa;Mh=kRiT9mg;JkF8=je$s9UN$l(2My&c9Krclw$Oj zUx``hC}y0*EtsS{rBB~JOgw7NIE5W&H!r7kb;CwsDJ>MH)DgeLnI@&wcqkxM?CRG( zrSNZiOw=IZSF|F0%U|iyDe2uZmnro51$TIY*tZYr%4H>$z5|q6IvKT_UkFo~*&AWuz8+tPgHNh@?*l04~Qo^mX*fS0Bq&VOHoHO)Em`Lu)$S_V#nJz-z2i ze${a<@FzANzFlIfwIr(5|53@mP$lz;rWx`3v1l5bn9!J%uf$x3iM&f%*}0mmN6XgT zf(A7YMgbzk!R=&R3%Xx4!8so2!)E2-y2L|lQY)--b+PHLh)5xk!4bxwu2(F02b~Z{ z11D|^Affb=P@|hClQWNlmot%1i1>>wkAY7tj>LI6hp_d@NDVsC)4C}nN@57bv5vW? zUk8vUiO@-sRzv%dN8B3OXMA|OKLn>p4E_##TwW~3Rq_ni{;7%>WQ@wT{9PCzlc;XV z@I_dmZIVQg<(+6E5sdkS-JzgNM74Z7S3q09NxCLa_rW)wg;{L!MJ(7dF0w)@o*|8+ z~m(0ddSy@_? zHBG2Sdn+7~$=kgMg`L5ZRTf}RnKeOFcj4a;Y0y(&vt2iFfu)c=<3YF#^Ggx`Mnykj z{f}m#G+-%gUBmdARPZkCYQ1uWRz^(`ujtg>itdqISUf~Ic)A`= z?W;mbHXAbRin-M}*2xk6m_O1?tQiKfY}AZGKrdB(KN&oC9yyZ@H-RKu$x{9?r^he} zsFNJn7A>{T6@*pee3b7Jb&o4~8|rQ2l2q8nAJM`M9=-wkvwUghWQBXe7<1C9VTxp< z4?YjribKVEJ>u$V5?)GR&e*=(C77yV=T;V>H94l{6+{vigT+$gBSM^qRifggz))h^ zJ>5bjDW3Q80bkb?I%oXpQZ_z5tRfc@oTA(43`I^q?7HpW3pkJMKG6vYvFr>p-fSqs zo;@oyc<5lbrOy{D%Zm%JreoM^ek5?H)}Q;LaPUPXbm9FCl?SGJ?R4sqzmq?khDSs3 zK&3|cJNF3YfJvvt_xxF;dp+vgI5*4`_1Y(kdEKfQhT zxwA8yCm(v)lL{{ljvk)Xro*GC1fK#M&Ay~PCR zcJemR6dGfCa1`*R#4U&esRrSY7XMQM*%-IUT2%ywA0wM(2z_6uVUvx%zlBdXufwAL z%8vVZXS}PUvg`uP`d6Z3=29<-4(58(pQaHub~JJn6oH|nwAE!lxy*GiI|df5wRW7@ zwSMFNrtkgTDe}8}Lgc>ZbaIuyN4Q9R_I-%3vN9SRWO{}79l8nsn^=tr<=gfCR>Vcaxm z@4H9*NueJNr%f_o_;g?~^b+q9hvmmnTC;gu0c!zkj1FeFHc&Xv zt@o7RTu7MkfHi;D$H=u zdtNapAgy%(@$7|pl(t6#^A9TQ+P(RHsWLZy$|9$YeEh7(x=SZrq#78*tPzY{`oM^p29AmrRS9kdmo;n z8LDV|$Xkw{ALm0nlKIY6O3rp)`;Hrbu(%L|>Mli!cTd_2e`7n}<&4g1ZH z*>)0M*)O-KW+*lGmEyu*^OJx;Rn-&j&9v(3M?oFgjExNahyb`HuQat`-L}m#HxYZo z58vUXuQ=Cf(d;b_IiU7qD^92Jw1~q?u=&nrWDs8K$*0Xl|P$D|qCed5C#a zof@tf*DYEx+wKPRNXst{{{n)IG?AG3gQdlU{3@OJA~32`U3Zl&_k>v@Sq0^_APhj_ zdir1KWkE<`>dtUuo{X|{28L0Oc zFdT(oM!whf7$%f(H7+zjHYnKL+dyzbJ{M8?gIHb0x+xS+g%4EdbV zT=5 z()zX8HODN?B*3^Qsg7^9l z`ibu}T+f1)6bv{O7vAxJ;aN*L^2k=+RmbvJDx@LX?u)-i)F1YWjzKZ|a>Ku`6pLYP zKypH2qPpp5C%nsV&P>3roH2>qnYo0Pp+^9_;ZnZUALI2if2saEWT6i~b`A>UOO?BV`v&BN86T zSjWe(!LBSJw`KoFW0L-s9?~0jB4$T0(4I@tTfh7nj3aZBRlWWkLixdkBgsm-^_fld z(B;c8ki0E^IHC)c!hvlkOd_H&Z#i@!&{_9h;4PagIvU&} zb2=)lQx&!r@};o=fkkuK#WU`}beqh@E=}39+7+l98RmO#bp&Y-xSPa+@T54d5oYp!X1wtPgQT-)p=y-h;3}3`u ztKn8>XI9bZH*6*yUE{ju*Un5tiU2n5oIKMn_Lw~_LE!?#+M9(cVWh*87!JJquV+6Q zv_;j3YGhfNm1c^fEI+S4q5iU}V_CS#_L-75x#VM5Ye1&-%*G24-TjcrJ6m#)I5`wU zM~KP4qE7A_dO6^ee%o?PTlXV%SCREojcsV~*k;AVT^J!^^Pk~G+GO#wC0MVh+7|g< znrtE%&7+WQMp#x{-93vYpHIv6=E{`lDG(9uwpebL?TFeaw) z1GSZG%hp{ZrR8`cyQ9%;7fy1N!SkbQ%&?()2{u-?F(Q$~cZ!y~-i~J@$_B)${p;J` zkI#Poy`szS%@aaD&54l3j4+9A!5A?ra-a*j^Gd|m)|#H;A3fLk7V%xpn}r4f*tjCd z=ubc-TPfBXO?a|Q_s75D3gDJ=dto01a*WEZWBvk$mMJt12POf5ckTL^+L!J!2Aynp ztC1s~w-M6#xyis`MI`v_0EACJ=TUWBhNLEZdFMHG*fbE9+-vyb+ZZSBQSPPwQ2V2Z zhQ+@>kQ~n_Qn6l5IT^?FzH0d3R7*$4QdoiqlXzh>W2A34YH57U;My~cy=`UbPuHIQ zR&W9nU+P2+M#&}Fv!w6;b5{C$)BvhWYrk`P*_F7_{$If4_!8Y(;GE0jm1;ui(S)Bi8#*5LcR&8l zSKO;>!C})d#PcihA)Ak0Oe>_dygpl>DlXn0aqkp!_|6s93t=2HYG%y^cV#XO3*_y! z(ml6@9+8!%+R$4uo600*;~37O-BoJ@SbV3SBEAKpO-!*%cYOG6JCa!(5J z*OtS6tyjh8tT5M=W2}K=@;N>5ig$tc?Tn5SbJT6L)k;%t8HcQshHi$b1V$9q+{|`A z5vyp^!m{hq>vaDD(h9U-wijP^A;1Ds2HQBOF#kAQHUdq?ZuX zID1JnRy0R61xn4L%_KkPWAo&Lw*7VdW#DHh>g3b3A2?M8v^BJDi>i5DtK)IY^J>&4 z%}T~B-LM41ehnc;B(8^q0#@IAi=4nDPPz0I!|C@ACckG;JQ?J z{Y|hJ4e_KAyN0RkPTw;k+lXrtx600?LU=dq_xC%$4yXPryi!8!p1U@fBa+?*9?Mh* zW+VrDhV}Pqn*Qid*P;5gIuY%&+ji(~k z@YbF$_PS|_%5v8obsh_ljSeK`xhl@ zwW*yw(TB$O7J7*=7UKEtZ{J?Clmo6WcwWbTy}u7!n`7>K#(tDzl;{S7#x~7T5a{40 z@f@Ykg=qi-6GnGII{YzYu151j9Px4#7)}#1lLlRnI?>>{cY9xA#qBwXmyi%3r+ z{h)tudvEJ?qqe+1uq~<1XI7U^b5NjMpbjOO|41h)khmsAfRJF6S@aF@T*5=*VAmJ} z&HmNx&E<(rHKA?PmK~lXV$8Q>;NoWoW=MB~`H0|TKPF5UH9V1uk3ne#SE0oBLm%b| zpA5cArwfV-@@ws+xVPUVdABX01i{_?CJ0A3Itnkaks64AU>8c_o3E*rvhI@Ky_QzH zevJl$5vlwWXRqOI&b-`{mZe5*i_e$aJ0ErPYgyPoZSs?vtsIQ zCt@1EzgyB>n!w3u+z97@`KviQnq&DRL1W$UGGxEsL&`;d^v2%Hr@}T;q05}JiqJ-B zx?R_(;`!Ywqwn)?qaDdA_jmQxaOBCEv!lM)MnAEzYRD3}PTW$j5;24t1~GR>UV$0| z^U-N~&g=XHlgrUow1<%WUhay>HHo|a#$2OH-5`EG+jsXg9gW`W&F%gu`c&HS7>zYx8cLdB+vplBa8V_*O({Sniw9*-&h88VLg);`d zy;#Y|0~db*^%Rmt0BmGkGUkCxtWA=(@zecOiVK#N{MkkPkT&j8{1uF$+}4eqFBNbw zw`QaZJx97M!h$j-n$(!a&TMS|#!TJ)pKEu7{sjxA89=7aiuVS3IhRFtFD%(4I||ZH zSCRvBzqS>zVvb9fbVN!SN}0jDar&jZh*HW{#WqQF8n0PI)5H`Zo=#M3p$!{bpTkAA z*lyWF7WD^Vhk8deIN@lUiMmdAu*uK)~&8B~3g2RtKRN98uBF^oM#lb4fo`&YV zkZk2MwmpbAi@#^m6M{myUf3A$q|!Fq1KJWMg&Ma4{<`zEoS z%YoB)6GTayto3yQ;uk$F;`kXu$$&uHwntyW6Il748^q0h@Hb3r8;pD-la4U24;Lir_F+PowsOb14iQn!dgM zo$&wBh{fWTMp^D${5nZ5Bw!yy!RC&H2X2Nd_24(a=_jw%A3nn~*cC&;`S1767O4n}G9CmN>fTGKxc-0aXQ3;}T@3nV1BW3b{wMcC7QzHWA@)f|?}hp=m6rE-A!!;n+T-0I@^_HU$J9r6tL4rxxMod(h(}(x z=GVA42X`MAtFWSJ28B2hpOMN_h9D)0@Gt>mbJPuC;kiv#R$J$E%0^%;e!C4@M5UtQ zs#TmTJo}X8{2CxNO=Cz8vusxt&e%RCk7#-JF(O|Y|H$S+DG_D$i_?6N=>w0g(~L!J zDu0;1MEPXuyrcsFKY51<73~w+#R`T3$wh6quNqzBlLU#&-DFJ z`ZYxt`A*T$Z~;1?H4|;>sZSN|PR~|uc_V7!(WZ2TYi0=80XJGE9Oo9@Kw7Z*`0-*y zd(HVFGnOOlZYlqP+;r8@kZ^!0!c9?xJy8Negl3u+!&_4(P@c=123sg%8g2bPY2Bg5 znPnO+Xey63lY;qP%v7cc`rc{J9VSxr3PqstrQ1Y?R8N1d=HXLtOAn;6d#cJQcFr#mGjn7E{ zf~rn#atJHaC!mVMTdF5CMa9JD6*rnyQ7NP^Sxg?yrEyL(eVq?K8&b^?r1kVnPw&Q( z@D1lHPhqDobvit7XEN<7{TW*`C$TWXQJVJ1TY`bfjUy8>#k2V?1L3?Zm}ts9#DR#d z`N>xX)1llEo$Y}qbmCK&7OK6ScwFvR&M%uXUsi80?U5J z(&oQ1{Z@=w|DNk<_*2*7?6m_7Fwzq18SjZC2TZd-e1lZHVs4$81@Y@(WBE)n?J2Wv z-u0rSJd<7Ln1=)j$fvAA>3b(B)`urepBUS`mtyX9v{X_P=H7uHdQt}5k%>@_gO}J(id-QCm;`kF1+1xb zY(wgwu&5@t@+>)Y4@`(fSC2hT-(=z%wEN!uaee2^t-MMlJp1?-8ToJZ=A>9}9 z@jpYKL<}Dbfbr+)%{Gx`vld0T1^_}W!rt9D|9W48F7UfR=?z>JXuTBJ^eo9#z1!Qv zM4EJjT}toQyz}$okM%Eq_h-@S*C1?2`6{Rkz8{X_?kD{-=V6Y0{Qguev24(SmO=1U zURSXn*012zSC`*`qz}rRTjcU`wB;tjR;8#B0th;Kz#UVEJ4Ffr?6P^2F^Yfs?5EQ& z$4=xEVH~95*fop?ViP(HKT=QOVDd;Dr0|(nn#5v=#WaGtx)N^=$%b+DT+-3zkp*Zn z;)_cx7Z+i%_n8 z-q;CHEZ7zNQywjQOh;Rxfwr+ogaGwBv;}*dPZeQvWRW-Z0>@?&HjT-%BQ&PV=7-#U z^edzY6*GllbME%sg`X@bqK1lX=q}w(0YXk3&xmBhFFD&^ye0TWT!x$iHCuc~Jd{01 zHDH`f;9scj$g6^ojc2?w&%}+w%h|v6LlRDd?ZRU&QL-=D_yd`<=3o>~>auSGW;Lb| zFn)MO;Plyf*C8fyL?nK$1>NVdu@5u|AK|o7m2!DSmF;$xwO7=rD37}s$L`+=iz%f| zN{r|-K7)<^0t~uO5v;Ggrz#ZtFqr)s5LQG2QQudGw6S<#E0V|ZWn$2z({a++nEna< zUYX0OvxAP*%%t_*-H8U1vQrkWo4LdYuILB}D^w zhiqgzZ;&|%4R)+xcUVyFY)O6fTYqPvo}1NJrW1)nkQPxl{?*|=y_K2CSE094r?AoK z(A=jL>Q93lt>}(X-MV~f5tG&$Qa;}~tQfZ8-B7vTH6i8CjF_8TI_F8Zz0&Yw=D?w2e?mfND0}k9ol>}XquDB z^bHqt8Ff21iamFg{n4zm3~d%T635^LzFMe-pk?hQl0egnM;m(VwuAz$MwM1kZj`^d zzmPGlV@0kEP#vQi{f@ME7=k&ZdD67P*y%^p-9TDn7v>ahnV`vfklC^m*{Y}(QX9VC z7m0jC>h&&6oDbIDN$^RnO}_-lXGzwK-5pYG-}8lAjbN#?bX|zfH113NfJNf9dzF9h z=XKbs6Z$IAN4B!VFqB$NAnInEI^Vv( z+WuB?y5q1Js>JqvG5s^j?A~91TmAflknJhl%GuP9P)Y3U@PMSXW4=L{Uh?Xp%^P|c z^=oFMm;hy`-fVvfe_VerfA5ioMc@sNDPV!rxLv&olDr`rNkV6))QgC5*A~b6X=>2- zy6z9zAM`)d1H|*u%;HXTWgo-)r%<|@Dy<4IoF z#?C@}b!){Gbp-H8H;w)t9sMhmaTBG(e%2gtijSDl;yuP%!d;ulTv3h69J0hxVO3DJ&_{~&Gln(MZ~r#} zxLgET^JhjFU;GwDb4Y0UZbamJp<&3aItPG z6$hst#amJ~Qln+zQ+uvX5oCyap1o~HF0$|HvO+Jf(I)B|Tf?SFR}n+He9qS7JCvmC zO)KQnPrP)FhLjo{L#{8kW(qn)&=MXWlKrL`Dzz|5>q((W#VnR;b57x*&FG0gj7>~k znT~;^7F;>Z^VVO>Katfu!C+D~<06~#VNqf!OII-tkE-*yblRXiho;B3Mj~gDjOi5h zE>ZCVH{EJ#A={OUdW8ou@3R;~+Gf)qB~ESZwTb#Yb9kxNC>Tcz$k%*?d9d`V4}7{H z1u{PTIc$SiI8>X$62g@+>mmVWvPhYQsmrN4D1RDBs}P+nop&V=InbYCy%3b!Yh22{ z-ZxxUYvX-kpYInt`4JqoLvDb-La%SAmtZTyR9p6{+r-Sj(7zY{N8 ziV#o@ukZ;O`$(c(ZSIcliv(PQ&SSEms>g`tg2kL$rEPi-Z5Cxmwt`-#$|NidXU; zp&W`0N5vCxaR}~9G+#!xHfh+=^XK5NPSa~tC_VYCMb|QwPDhX`z zp)jqW0xfQ|EF+W{+KPon?_&CPcZ=RZB^#+wC2O~DyFwr37Ze1cM2nlCk)V-V!JD&m zTAcXj1UHDb^3>x+Vy4l{N<{oq1ZQB*=B;`7^sF-ii)!Y?x1SDde=-fO5?k@@)S@I| zx3HiKBPKgi&vo>%uv$xwv2Qd5EyP* zWNC;SEme~Gztn8g+Z*PIawwWJs6g}nlW4j6OWFHDkARfYR5vL7PqEZ#spwmwNZ1~t zUGRm-XE{)Ho@%clJ_~2MSE@OYi0CK(X#_Pxet&EXR3IZR)R{p}&cRfMjjLIBi@Vih zM`2FY3FI=$kKMi&at&uU&mUdy(}tr*Pt6iDWjCO91k+>?yT#;$Ui6^JJ6)ZIPcN;= z>R+C0z}#<|rN=iu%2c(@sMKIX_)THmk>sYWLawa%ldOJ+C1%N}H9$hpk6Z3HKc>)= z5jBs^iNP_*LK|hi;tkhJ9vxH{FZn|V(g|dxIGy1#Duru=(*EhRs=YWcxLfL z>>BL?Hw!%)i^`b2eoP3cu&7WYN+S*dKQNS(p7s(S2jW(~<4odNX~RutN6N5$jg*HjMh zw0Y2HE4!pDzD<7wJ)?`8Y^{9-Hw`(Y&Iz514&k19YKTr@M7CA_)c-|n-vC}EVGqlA80#7-0 zp37XNLwEv;2qH;K5UH?)QvHW^^7Y(u=SYKuVl8oHywv750lpSbJ459%~xF6Yb@<(CUPR$3VG8!A5yL7 zNZDuItscTNi%Sy%Omz6EE|F3np;F-oLR!=(i%x-@y%GnFcRJUJVP;}bE4>9hj1yO$IXE@`z{)KZj zXk2eCHqDonA&r#D%e80HII4CRq58#y8$2HB`MHb68szAxU19VU2OhgZJp^&ko9MYm z+;R0x!!5MXcXqOgF7Qq2F8~UmgANA;+yTG=003+V82JC?H7&3o$1#M#v#89oh|V#T zAh4exF!X;-qI}g;FC~g|KH)QZJw6DUhrKZ%Cm4NSh~;*29v+2!XXszXxTc{YQ?FLC zQ1j6rx2YW-Qn@_fT|KrS-=P`OPimcnXDTSKfO<{pUw|`DY6u82qTF_llHFaHTP^UDPt$dL&l$Qn2j|>yZi<9klmxB(6 zLddJi5<)c%HeZ5(Szlh0H_YteV@Vn`6lcRhcb7?WbXz3><1hn31q487*_wyQIKbo& zL?U-F9}8}OT7p0JLAHM8Z$iL~oEIdqNs-E-pIyZjebyX2Z(?L8ttaXYy1|$Lf*kOkxZGoUN3+$#vdO+ zkh3V?;Gu6i4nXz*ZC;+KJtVi=6zRnXl8*CHVKqvO_4MP`uJ%Z)Be^At2Pi$7Ec{H~ij;P>8t380a_{0W*&Is*&7z18G3*H z?-2+YMGlJKIobhTl*DL5B9BBYv&(>x`)2S|dBGV+Xi=O$Wk=HzPl_rKGWF4Yf`Mib zA9fJXANwC~6p2YL19AOa;S+c;q`ui0CjJn3*PAlW9;xvBNmI6m<W;I z0;9w+MFs(*Hw>Y3OtEl|FqP|y%=o~QF4@-!XieCNAGHGS?(63Z`%pemn}&ntdVx1Y z!2j|r2y*-{x&O)VzYto{j3uGSw?jDC%R@-N#b`%>kcXR=(2|s#MQIJVWI9cXHc?P_psaiUZ|5ORaA;Kx8-e7^laZKDPH-@yRCab^RVg|`5IfSHn_8bg6Ow$bH#_mS-(6!cOe+!zU;+i?dF-RRR33I5y~1DoAX6-EaR5&6#SVD8 z8G5b^&D0%SBQ@-8k809yKre+Z=x%;aIKzxg__5zuz_#5F%FF@n4r8J(J1@}AcrylN z0XE0;{Og!t(Mg)dGW_&oviK^>;R;I-WVJErJs;g91Rp0?9}OCx!TrBz7VjU2FoBR6 zo<*#G><}|stYv@1pW6ra1o2+e{P^5bj(7){LSDGt`U?)!}#jq z|MC7$zlP4#ssjIF{+qJ@&$Xw3ulc|C5NMuuYfKqjH3T_o-oR4D98{AIj1nCc$Obk_Jr;;0YAnbdJ_B; z3O%eYG8mUrLo&|9<(vY$-u&n!|MM^3;;+DYk$jsLUPk9B;j*u(`9zQ#(r^Jy3moR} zq|{^LVphXybaV!vLi9b5ONV2rkw0@zd`Q;2V;S-)jaC37Mj7TwLU%x2rsZV`u4iG} zZL+aHXqZ|q+Pn;fS@Y7z*Q9YXU*$(20ee#B*3StYm<8Ftg#zq1PoL`DOOATR?~Uy@ zPkig$P|EI_?C{OVF~WF5Fyv8^r5foPWUh^HClry_a1R3zhB17G)A$MAVoy=DudkPO zBz~dCHbiXYspQX=@?D(CHg9TdS4&shWYV--t7{S3c70QF{9TU9681n$B~Ja}{s^Mi5DbB>7 zs!7_;OeUn3@+c_LjgzEIDjqv(mCJX-TKPiBcHBlM8pgc7#Me^X;$hcfRi#u$-Yd@elx1;GHyk zC%7#uJRYZ{*~^I2{3(_;ZP5Kw$h#l%@*ir$vBb^d2b1kHI*tT46<>0icxCLoSi;cS0?BQvhv)Q)kT{7rX@W-I%?>i2tv zc@dpgnHJSl(Z`H8>uw?lti11Yf{VU6&a@GbGKABa_3(4FBzR8JoD@YTEM%0Im8)oN z(o22VR@wezQTC$jSz9c%b>0*)56_R|;&&k2%G-tX{UueeK5$b6?ayp{odwW6g@)@R z|GrP!G708!|CJv!IPyAi7xoa$>mr-o!nAe@Ei%*9Uqzd6lPWswbh*u?z~V{d$-ZU5 z&SpA?!p+zZCa>)3u!);GQ5?pR;3TS4FQeG1YSBThk5MFAow7~f1}zk98GSihrO`ew zTvy{#N%@?kHEv&jb`a`;ce=;vo|Cy2%iQya>pyYSw~@IPC+&e}O4WG{;82SxT=1Il zVn-QYQ6RzDOXvJcb&2#@k+9;Rq*;rP70ItINk5Ck!5c)08yNiB_4qf<%=bm`?Y?#jBWJl1-lpt!7 zyVz5gqb~00jg#~{cKXh0<{mREMH5;${=A{=@KF#Q7|ne*!6xq}elwp#3^4d`Nj3P^Py@Acig|}r z!f3v{+0 zpypyJrbJQSxk;9d$fCd#Oyf7|hy8HEh=CDx7LhDTqE?SntZNisS{C~`E!?bv_k#ps ziJm0kp4X50U*f;Se?vcpufgiH#@&%?NNccipQMHFwz=p9Ss%!e5;M76l8zMNl&$*U z8{|?m2!}hC_4N=4&bKDllx8IGCDv z=XiPeR|Z`@mL>^bHVH)=4|$il`FDkJxLSrTO1F;D<&8G#-0WgbOy(*s3Pu5_#d@3r zDu|knP9N?!^BvnZ9CHM*FxrXd>4k9(ZIW?Z@Hg?ej5t(E(^xxkf{TigX$TV&BMzc* z6JPZ3qwDzgR-=|{&YI+?DcnPFO0To`)z-NbA~wTmwN&hdMrRyTLy(IOibIg&!sL}J zjB68Y!WX1s<$MWlvWm`8@~|D}oSI2(#*#hx%I6GWRVtPENT-|5c#>%y=-NI={Lu3t zI}PEhG;U_6R=b!D4~edgMjafeYlV6N)~BR3O*VEnwkM?#qw}yyKXGW-kO79+s~H}s zz|X+vsbxC)#q84)3OI!zOJU34OQ7IzjN#45nU=0tMTS$gE#=}VCT-ekf6lnbW$ef% z1j$VDjxX2|x1pNJ%gwl)PNzCWi6bkTBUAcl!X^o@!lgw^Wm3e&>>I0h#M0nenxm%S z{)ERVmn0r{)qrIsj^1P;z5O)7gu(K;exk!2-cBzw1$#b13j2X1i3xjhA#wt+FHTsa z*hICHygkA)irCKJJh4I=jj21N5~VO_^E2j(snQ-kx%ee@#~Fq7!gyj-zNRpPUe71& zF^^_ZLMK$`_zgwa6VLjM^1nsNKYUymM5O<~asO@G|HH@qZ&6aj;ToJYzk4&#D0ch5 zixOzue~XecJo7N6l2YDLM)400S*;oFu9#d8IK=0;=?&(OX9x`q;2KL{TNF%z;K{WJ z`Ed!T)qFc&_ajd>buY+YBU)ytD9SZxpJ0^iw7+SapCJL6fyZ7cPI0ZU<|ntV-^Ra3T99CQ>aZVTB>79YHWBDwSwyWEiY_}f&SlY0vv$797F-UG^G*o%Z& zdrtMk_m$`OxGikGd|5lH!fTKD-p#kbYwJDVeDLu3uK@bIixNt!|ExpF~|1xUX19TJ7Mu%wx4 zjN$Dq>*Hhki=SeEC>>)&dmV3&W#=2P*r6Jjvr+$?Fj~wSLr9^zM{0Nc7&s_ENCB*{ z3-oq_Z_M(U;S`GwZrAw0z+dmr5**4?0!7)0AWLtt@2{IPv+@5C4z6nma`^%8q^4ED zsG^ZuID7^*-K8p-{tM{#wEtzTc>N$`68L@L`Beh9`{cOMEpFLzgdd(FEJ!Q-h)PX%kI!f?;sO2mVm`BWV# zpN2kVTV9i{QphX$R5Yvj0l(*IYT8(Ciu;DRp)5u5SW!0_mMU1=wD|@$6?jx^p*4OP zDQXj_KYmkjL^w9`x`svXB-E^RfnrQnJN;rP%ky3Rcge5h`L1M#nvW%G$MGc}WA zKOsFsp}V*%7t5_2W6A7iQ8ixNt@2vZfLIqV(NB2Q>z-d##|x4^QU^$(SWH<(EgP~p zMC5dNkwS5KojPrR%2HBRq8YzLLRoD|V3d`WLxw|?L^A9A{~w~>0w|8A4HsQxad+1M zi@Uo!Ebgws-QC?Cg1fsza0u=W!QCB_@Lm3M>eQ|3+1c5u?&+TCnSHC@N3tM94HfLh z1d;>xs36z4x8H*twq{6o!#wTZPL6VZz`poc75}f0i|XV@`(tcyb&MiW6K3LYk-)cb zNKj@AD8g}kGw9R+6LDgwYHoS3%N2`?8T2bXvf-bO@0x#^iBhoDQ#ilB9d(tMpwl}E zH+LmEcIl6IY}cD@-Dt61zhdO%|8`oEtuW(a()L6zCxn4ron{JKQUrUODLM zvo#oPJzr1JSCOlZwGlxU@U3!{8lupwCEa)f4PR_0grs|7r>ukmQ2ko@oL zXZ~~Z|2w-d<&F9pEOXZ%Qr8!ueW|ZR$xK||nvCxL_ut&r2U>AdY4VL9KPTv3I$PpZ zgwGdX)dgPAvDLJNdwLlEK>O=szQF&11|j;=9mcOH#yw)AMMh;t746Eye@)i0uC*Ut z87#8eJmgJj9BE05x)o83)Z7rdl&xw_)tr5q(d^iwaAjYOwNzf;^CkSo{Qq}Y8A}x- z7yN!oB^@ImL?3(k7;b(zT%g#90RKiw|9O2%hpR&M2}Oh1?toW3TpXe{f_ii>_f)!g z!f5eM(Qwt+-j|UGdy}naKd@0$jBbS`&`NLXz+JGPM(zLZzK&>_E0m*`U*3KwJ4OxY zEt$zE5)a$uI3+}h4~^B<>Q^R0Rx1&^!jdH>H355b=0qY`wW?FC{O_>xAp z3>^ygzTJ zAOHQ%KK=8PeeN~D*o%4RJwLGL@yj31^8|pBOR~uL)tCdHxN|6aI!OzNEr}4SSFy4} zj%q22oD^GZPn(*g(M{12&8(hyGC|g6a{WxT##aF+$(c$K6U54HrY5k{PxiG@b@lkk zKYF}no?PEE&ukqS<#vqxm;Gj zkWX6**26;sSE4YDlU|4;fq=>OOL z|B7w?j2%yXG@SoL{L(FOZGBCvY3`og+J#_{`_a)0Rq9t^e+xJ#rQP%$BtU;KRU~^3 z<$zuNwuq|ZD&lX74DxPieN7l5vAgO!KrT8J-cCbQ{i;E$W#L|MBd4)pVD-Q1_@77r zw+6ysko_CRmWqY|?9#Jt~*S?mf)>W5JCCuu^GjSG9if@~gT3 z*BdF1-8+=zhp#?i%Dl6;zJ`4D!syfYOSCC)S`Yz4)A5^L0srSi!PINcpx?)oI^@>pdV52%+9RotPd#C~cS+(zjMc8(JAllScjD@_p-L6U`w!RZpJ&!^@Ef0WKLW}tF&Vi5u*kJaB z$A2&7PP6l`uS7AHr?M|xUvsn8th4!7|AUYW?*bSHdufiXMH$rr-^l@^K~Mg(_u8@( z9u>`Q^L5w6b!(4?fWdd*pX2W)`_E6D{#u>&v}1>gcNe5Y=}-f^ixEqh?Yfr!2aUEB z8dhblc*{*%0-r^>J?xA!LV2g+I5_LLCJ(k$xg2AYBb(K5@bOplcCpaCx=Lr7{N~`6 zuoW>cN}|f>f%c1vALCHAP7;Szj*YTvQ&C#f2F!(T_-A&}?jNf>U&N!o)pyQ3LNZz< z4R?Oox_fAgc)RCFPV>z!vmA_D+uAIjA0->@F(O4YW5j7;HL%}{q(FzIB~!Lstcpm_ zUEo$Uj5Va1Oh3+Vri)f?rvqAOM4cC z=RXtk1o`N8-T17AhBU6^BpWCk^sckmQ>Wv9E`Gx|qCnzxUJl`WlSkL}=T_aDS&N#f z$rC!OJ{`_}4=ku^Z&U}CWDH7PL-37Esc4uwJUC;Bd=ujfw(l9tG22ujY_VGO0H)`L zC=#T+^dR*3=vT0#I6}_8K33U)2&$V7y8qs&EMZF?iP>M>E4)ZY zm{3h+vpFDoh1C~YaYn7VRAp~wd}lO{LZaBzFJ%=I5Jz;1P${+k-zCH~G=)T4M)(D` z3L&8oV@!IC6`bdJ&AmImJzz`V4Hf$n_8F6rmY+nk7g-9XDz#Zcud6rv&d|C-9JNtk z8mJ(}5?ShBG}1_kO>)e1C*gAfZPL0aZ1Vm|Y1w6X_z~O8j&nqEISlOE0)3x>9H!w9 znI^=NSA8SCMC!V4Xv7KGe$F|&Sw>!Q?-|t?rAmtDhr@cnTOjC=E`u>l1|Vmq6gu1+ zNeeoP3Fi^R5m-buzu`|YhG}R=vQ8rqB}!p_(Q1U{a3{d2m8B<*a(m|ISkgUL>XT@VQqbC$^dF!zGtfs1c1NChXeiKBp=6I%byTha zCeiAIblCLeV^DL|#wGw$8*0E_%j45RT&bA0aGThRh3ilV-l(Ppf8f8iaoNMFxHFDg=qcFTeR$Oa`j${j7R4T7Gl9FnSDF zyp4THoDRfO8FLh1YaeT{0XqwSVAy2eEz)IPU>?VoaX zK-n2u*#5+re%CcnVCqlU7RIJqEQ;+c);{ka1O|0bk9M9EfXZ4992?KN#!y2}nKJ17 zlP-v^$TO^VIy&{^W(vXW9*UpFw_MBVgS^Fp-;4l4J0jR5N`|LUd>Zk#*=BSZ5i&fA z)^FI)H7lcG{@XH_RfqQY*z&HX^f`*QRBIlpUc>iK%8!D>g601JNS_sN70<#W!d0K> z|HQNZ{j9oAyikFL4DZGZ*Lj`02<#73`?vHFP1P;=q1vq9vSPiKX5dP(f-P4)T(FOq z=p{!Z)xHF3#a+n|k1GXMUf`tmSZu)qX@FGDS@OCxJ;*KynpIs3_*`yyeR z+dK8AELT};z!quk6v9F+iil*o27AC#FioySl0aDYA0X7GY``%)?^yy~^6UN^()Gwa zu>FL&C{MyB_O7gdCe60?RY3A4Z!#a(G1*g#fH+|!jIcR##Hf**20liy(?Y^)K#yQF ziXqeuE_kTdV@byCb4y0a-4HfFy;Zg+Anp1w{>p`@=*8TH6%juKXvXAvW~Xlq$1)WnP^-?4oCFx1o|C#%64tu| z!wZ9YGHs>SE2S485$xwT$tQFWHgyFg>%f2~mFAoG^s00ugbpYil)Ez&cxYIfs$kK> zw6;%7I-uCU{nGhjr^OpJU23z>^n`m=z|J$B5jS@8lH|HdDF^y5WP)rau_Y0a?|iGw zXO?L8U;N(-rPEz!J?JKTEakq!but6BlnmMI$z_ieBbB!^ne% zd-DZ#WA>&kY_l;7{lee zT7O%aaU6_Dy`~77E$7;=_7RT465NA`$YasD#a9)z2H@>XcEY8)(5aq-c-qUTOtHY6 zD|9;E{KM(u`jDI6_Iz3_qdax2C}8=Ovvwl)O@cI+R@=g(kg%8{@NK$B&jCF0TarAC zyoyu@JuaA|8Ziw%TIR__#k9_vBpTC zg8>?AuNAzKA>ROnn@J;2ThBb0E}~I2h3mDA5mtRrE=*(lj~B_?RE-u`gE#3$cFhj`Sj|2CWb zKrAed`s;+k{)iU#Z;WCa~U7j7Z4&2qR5 z^8)79I)ra6iVk25R<@GY6(a;bZ8`EfUJ)EMwyeapA??Z@i5&JRfp!(ZazbW;l{va z)AD}$_ddl$wYC1?!1B`XIq7jB}pApm_ZB;hWrs?<@i#SCL z8;k@Y$B2b$mhYSz(+bRXA^R+?suxc9EYirwd{abbT!@OT1XR?8#x+PxHBjkWQf1hi z-B6bZh%;X7M}@pY$TM_>^tE<|JoA@^D($d(w^nAYy;Le4j0@IJ!9}=5&@llS-Mu&I z#?O(^mS>35+BXMy_nzABO7P+!<25k8KP8}M5L?9IUn-fU&C$OfWgb?>mwHv7kbyxF zVS1wak|!w!jR0)J1#lL1B?ySf=xx|{X-Iofzdc*XGcnD zt-rdB7Nv?}u>Nu235}&EJUw^sM0!3+<0|=QG_710@+D54w$ebgc{n25NiCPtLhetg zPw$R9ky!q?rs2E)ULj-|;7)lcfFe5~2uXgnVAv^$fR2&pB?|4*@@G_xx1`*2uEj6| zHDNgFV)m6YEkEKSzk8g~xNQ(@rhz1xs$Fy}p2O(=t7-o?GM^38ew7_F7jMJLws>;W%nN=d8R+(_tFhL_4 z`=m^1M3K=S)M6WPjI{j~;bX9V;tSIvmvE}AQ>8O)KKK!as7cPG%hSf9q!eCLG;A zX!7f+Hm7^Vf7f5uRkKUa7irn&?nxyFRf(3d zA%O1kpGHmhf|D@QCR^Z3XD7-u7sJAR8_1x$ia;A;4bRhywk0&ipN?sKxj%F{0aJRu z1B`T&#WNYYcV}cUl@vq6IIW!xg>=MyCUdYh!bgcA!T9p>4`ixy^@44k^oD0_F>f+#m=#ou_`~V(anXReu?IczW zqR4IcOGFZd${%a=yYNXdj%E%MYu0L;Q4FJwmHNg>BIh|I()(z!B19<-T2W=Vd_R%iKio%cxp{^DiVeVTN{f`p?;};eP#ZeJZ;TUP(+~n` z$uEO9C8th&Vq#&u69=kJDOVBqongvdvvM9K(ZuSs9)ikNnESt(ohG`T1wcglp7V&Ck-$NIAiW>cl}nPMv7m z8#Qfcx|KQd^erD#M$#I23=p)i5&9l5J_7!>o*@Dm5<(m9&Q5#5dKNqdz4$_w0&(=A zaoi*JZJ> zq7Au`hP6+EfgF;4uVGsi=dQE~7w0(q;726eQXu1~CN{ORu^as;y!>8p0lS*C&%v&S ztr5lk3jjBR$6Q({f59m7q%CmRKj1NfxO0yWf?A0bUsg0PBrf5COea}c#p}?=S1Ltk zpQ-h04I3bU4Do-w6Op4sGP<|*7K{`3AgTTFPi9ulz9w+zkMckOg#cYIupfJ2Z~t3k zoXvLE$E9ufYbVIe-xRkPD%X3QWuffE#9KP{mNev0<2K!BnPNbLxF}uZDX2Y6ohNby zDLmvE!y}DixjqXm%tdya^WE|aHk-xPtn?PMzn53C6-zJb@IonNM*taOjrEs9+&PUg zqTvl`C7f2R=;&z{X;__U+bVD-*?&AvN3^nCB-cQ)_23)6rEWZlkDi2fX7*kn7}7y0jyHLDaGGwS?DU6SP z^qv7q^4#J-0Q34=He24=`3Q&yQZFDZ`|!+30Tx&J>wi&i>nq_}-XD{`Z-~wFEeKY+ zwTWLW9_)<=`SGRiB=@iL`_q}nC%MgYjX0{EJli2K3GDU@rIj>a^FILGI>ZVS`PJMe z0h`TTh0SR%>*#-g{Hx&nVr8{&N9kPjbEO`JD4_{N7x4B64RV*8+{%bLTBh8W3>#HV z3H}axDtFghj#IB3-lS2I2tX4rKK*=x9p&qf!Qa8;i7r%SpvUaHJT787=5B^0qWF}8 z7ySbxlaO<)HtVK~MQBml5HXKAn=%2lOXl|ZK{*4L)biMHW}5UyRSAXU(*4ljlv6pQ z^nXbrr8G5;gKL9OaJnMU2y{--kV8&g7}>H|B6nts0V`0K6=L(gB7Zma@~rq#tB&AwVqwp&n7Zso4N4$!3D%e6$>6_xk@aGq zo*FI`^gSaUOVO8P$MM{7C+!_%PS&d#Jegzw_H5Nm_V{83eImjnZcI#NBdY+gORvp+}BkVM~#B-j~N8jsArkM-EaW$pfySV&M zzJz6_7}gGcBeE(%&afWhWMGt-D(ZM9fkgy^5`qR9FB-sE%CkQgOq^_k&tQXSNX0cy zOm5-y_dfud3eb&2?a)U!lTRa^HO5hPIskq?J&3r7qleaq*&cQQD;9G0!TB25HrEe# z=c!gXcI%wi)`vp`+ocmiGF)<;w5bh!Y}ww@*g-*u1wFyswiU!O{lmkj0>50YN6;yA zbiZ$& z$_mNmfwC;TW&DBAb|&G@0;`0xag9(Nc{kU2pQo5SHisE%hyHyUSpiczbL~M8y?VKX zl(zJji9gxptX*stxp#A2sx@8OY>(m{)H%u5f9DKeQ#jESeadi-m!Dh5HkITapybw1G`ye4B;kJvUzco zgx_}4Z}+!ctbBtQGyc@TeefJcY6O%Qz8R!aFx}|2b%R4e;jAnHgzb6fx-lYKNM8&+ zdYd+aw@HR3VTQUs#)^r*`pPo$z8{NZfkM@zAm$NYK!ktlJt4yhqLkw*RycNdm)3*UgFdSvK)Q*==s6B$~>SAsFL@wEKkFz6x?I z=p;`>tF}lkf>;>T`OZ>NXQ&|z|8_r~<&IkChYNX%@MG_fbgTBBi$@;O2_wH`Wl<0| zjxLBh+l&q?25Pc}y{WA}v1uL;P)thxS1!3P@Q}U4-o%Ip$v$rsk_BQ95QYIN+dE4D1BZnPST)}6 zxyB_rt0d?Kgij=uTZy+$F|SY?P;L|!^ZP9Rs3O5$6%)2Y}Qf{JErN_xLJs@~E z(_em(c}Zj%K@XMqmtww%`*6FT4yGwz5G3Aysmn-5;>HuTjCi~0f}VkugR+An+mhjs z)7m&{cJy5z2?DK%8@k-yQzcwS2u)dHH}y%*2+kXx8q@9wSi`R{V(WCIY->GzC;=7Z z1w{t43UA_2C;7F|hV-=HZ2)CAhL%yu<20AIQJ7_!xiGEQ6#`p>lEom8fm0JR8eU!B za{Tfk8&U|BVO^0$HOH@eb)ykJUL4Cogi1};NItxoHs(Yb9Te<(ToUnUU}7bP|I9d( z`jya;$f@?NeYNy~f_yToW?<@Gz$F3++L&i96J$N{D~T5($jza1oi3^@MuHqFyVBrH znbvtl!m(h{h<5$qOQJa+_%DtlrBj_-FawtaTd1u-xGNEDcA)Ej05KLY zysM>h$0m$@P%q&5#M^B~UqHw)GX%<`F#L%5f}<5nSFFX71@9JNQxB}a>h}{Hf9^ zAtpx?I6WVhg0Zw6S^lf}!^Uzw za>)&jff?O%U$GEspyVu*@E#0+T}DLLp(M_x#y7E#CXH$@s4|0uGDM0cn!$?53r7? zTJE@%{Z2%5iTmXXb(|$^2ni05B7MO8d-Uad46SP1>XC^{ye~@%&|?p01q&*Zcl?M zXEWtV?(QtaAX1~&$%?z7dSTYs>>_@*0w-y~P&fL4cA%nKG5yTcSLxQ!VE*A3NmGtZ zY(nEU7KlY7XGqoMyD_|!vs&06lLWQDbmqR(AsZj&NjsvDql8tH3}z=-qOiJNC)WNR z9MVFeo~6(s#sz;`0?0!ULkJ+XFq@LXMuITKMJrj#VVzLPM58)V)SGvHITv}7Yk2UZ zD~52x3~IARU?@Q3M#ur(P9(5@`4m#Arx?ZD^3bP#+)3t~M-JRy70Vqnx0floeFh|E zUEII#%#r4u3*nXw61o$Q{rcOO8`Ft|*te?GSEPa-j3Uv-{41~Ap=&y)tfjYB^DI>x zX_XFP`zfVKEa-fZ9*(66i=+4$BnVz<>h4`LLT%5acZgq9pHJe`R&8h^+*BWOw?3dC zGfup!AJMAMpRNHHSp22(-_Ew)5}(yVQzy#woRfX@^(O}`Y`C(O(1p<+unZHR&7s5a z?8CfeBG`rz9^l*7MX>}QhafRo9KQp8K)dI7d@fv-ipte`uC40!&v9y66u z!pO)kBz|pQ9@12p3@(jH&iHXE_{JYgmhKFRbMvo2)@IBgpr{GlrEh z4wp`=icwvf*DFOYiI2$HK!KqnNmQkaYOVRj=zU3UzcF3)L|0$v9ve^>ZlILvu$nnx zdNG&ExN^x07gM0q2Mrt84Y4{Ne2x<5!E)|TE!5d9lrp)B7b#y*@VbLZv}x1BE~%p` zXnE84=7_s$_vXB}3z>{ZxJ|tgfoYWd-2KL*toxjh9YTkh%!tcv;RcQryxQ>OA3b7QmhUXu)J! zw})0I5icV+L((>+K*k;Av#m_DF4sbZVN5G#3yS*1D9%CpbK#NLP?*2#AiO9I0Z{{K zWqJQ`L=*20PU|~>FX}67;=cB0SAIUpe75TKu>$IBrjYcNW@)wUH#s5v{I-|581H{; z>Yru|xb9H6Bz{J^H(M<5l7{udc4UCR{5P~iU#-E}^h@Ch8{_&Lo}TeG6S7u+jZfxk zSS3PtQvf3Y-wXKw5dvZDYN4pDWc0Fy{+Jx(T+_=2++?TfFse0OGYBAbZ=x}9h8v!9>=vY}0sJXTP(bq5TR zk@g(fSTiHef@5#73oM93Yu)m0RIkl8tHcsk<~)S}^@mW8`{_++SY?k|oC4%nut<`< ztfR8GvZIc#2cSBv*NpPZUCy5@4WQVZb+N z-;xuBk>72SK_|RGFu0>sS`%@w7>fkf@L?napOX?D$m;?~333Y%Fy@@hhS8b|DPm;p=59IcaC5Oz$j zIEF;hNJC4l5$eWfOl0Fzmzwe)qJX;auUt)ulWYH4K{8wM$t&DXGF|r zIpoiX0$Jm!I<3F`J(X3X4lNaF3$W_N+y{h6YK?x0@4-bV8rPJ_COrHFn$TgRiFRYn z1(p>-*zA$}?L)Ulw+!Qtwu_VvMprDCHMZu^&PnfNR(i!>i(Z0utccr<)A-35oH@0+ zHCNOvXPcAD*89BXi)_M-sxzHkc2T{)X|@*e?QvqMjF2z_Q#ac3xT7W!s(J0&-*K`w z8j*h}Q~&m&_ysXIXk=-E>pg*A7UW&sf*vHPe6h1)1SpKx9>gY&Al3fxWtyR}$^x$i z2b;S|Cej*HPpkq-pq=i<2uz9joNESiGX1(o2~0E#+w7dcFZ&GC!4V*Aws)_um)Xpn zCL!YdcsKSJR87(o7VQZSBqF-50hY>?M*(QK+I~s0ZTeJhlmK(0D)tUdb&pG8(%(!9 z(TqL%t|<1J{LjeA8O!R58&N@mZ@!aHvT!CW*tNQ~aC1`FU=9L;7S(rV;~ov?UuL(6 zv1Mr1*1XtM!{?5Qb&|F?ubB+rmX}Nrj=xmoyIl5Rr(=~4&5#VZ^AtpvGgs*7mL{PG z-Q=$OuDyqkjj|t@x8HDYaQ&FQ@BYs>96x5SFN`B!Ab+n4687$MF!uxir*JpcdDkLt z4Oo^vi*t-cMTSsU{G5}CC8K-8o$a0{sffe7iWf7K#(>?1INH3}+pOs-@k=huVj{I__KzDb_v0F(?v-xZK!Avfc(3s zWOvQzP~DB?t|v>2_X-vA$=5neD6|>sLU`_9pS-n}hgiY!bktu;&3#UXiaQFlhO|RTr>7sv(=ZbAlWiljTOyiPDU|L5FfmR0u}4KYF60;F&Kjsq1y9 zj_#LPYZ*6!&ZBr+rGw$X2Nx5hNH1cqZ;IN_sUOrdE<1fwZ8ZD zr*`(L_=fCXywoY@HTdo=*0;3yL)JIDOO%JZr$JL!8MtQ-#>1qgL@-9E3+W8Ly30Ij zC=h;cx`4`G2#~~zVClaD;C%OKkCMa*O~+wlqtagZs~mF{xbLSE$4iDRB#)haCU0IQ z*W|U9w|{a;WjjSaBknzU0j2H;qeqWsVulcMF{qYvHYPtpXktig>tONXVW+0Kp}+Dk zE6_x)-;pxR`Vw~xfDX}!i!T5Pe|j~Hkk2i;gH4Tap&9D&nkiN~r52tU&; zvW15DjuBE^jon7%i1zv~@<}-;pkhE+ZI86lujC#z?34Wd_eeW*?D|7=8`s0}%onty z922*GxvfTDLSgYxGMOHC1bwXP*w!z!{O}Xmq*gVDm)KEY%XQc`F}JGvtS!r4s7p#KB-u zP0)jE;=o2?bO1SFJH47BJFQg-!X(Bon3U*UoCBS<_xNprhTh_B^jNG4o-FRCx z%zj9doc-4A?7Vs{arP^Z=~OFEEI82;=#nqbio_}wMTodF0x!R19ILcY#FMJgd^SV% zKB$9-EH!cy?#u*x*S9x(EQDz|6*NGh%4ZozU66+V9GPb=a|R{UWKpATrx6(n!!L)S z@Xn(8nh1>+IuC{Fr4AEX>LHz>kf|kk0X&2l^Q=Kl;HCUQlX2h!69sIWY|8ke6b%5O z*`!fWK8qp8BHQ#te_QXx=;;b2-zp)1Gli?~HO#%61 zvTE$HQQvVo|Mo4oqc26RP6Ed@mnU&o;59P~|ZNG*lJyV|@NA8OYk~9<&bB+O} z;h!4QZg>^lMh1fNwzaA)+S=9K+I3g{lF;$zlFr^pyCj}n#})P}p3!M8XKK+xWK>q# zshagRG?RuRz|fhDAd1r*pa4;-M`-ovG0d(@TiYPDu!4+Hs46r13v$oi6RPb0188I0 zOG}JWn+$<%U^N&YO~mTodiZSG4(hU!)RLXCeb9P?jiz2ymP^HpG`+}ze`_=Ro%Du; zC`#9Jrd^cPVZmB@Ne{uK>o~$Yph|`#KVkVL9(pn08ebTSO3+t6lr-iuq`_m0lI~r+ zIYt3nLY(fFgU?@T+1MHkTIpC6vNJB`s8Fq! zMFUP^L4}li&`x2m?f}4@?R8Z>?$|@G{-;xJ!goJMgY@&dS-#VgW{W3bL!AntG|8SL z|3uqY-M@;qb|Ww4hCd_Yxm{aj^qMYh+~2?r`@*>$wn+=Qb{RD?Q!DgeiL7+}Rz0Q+ zC8fe@xfcFnldndKKPkUK-`WnB_VusQ?X;n02aw-0kMz0*0gHmh+CVEIp@=bt36#pz zLC(5#CFDPc;9205U@Ky@h5N&FI-zjQ-3ihJZ6MZ?{wlPn@R&>&_euxJFe9JQ023PWkQu{p7%9zLG$d+eXyv zz|8BC%J?)!TMjRSN@>{ALy}sw3q-odom)Ccfx4dqkaBiorqvGW<~HE2^n2(n{!4^C zONGz7#X+_JbGKNlBd18{{{Zu3Kx3BxH&`<^((VW|T5Ma%^NI_3!^b5`7s6BR-)J}~ z5tU^1Awj>~xAnK$s{*srAr!w?Id&*^FBrXPSHH4|OGL*kKfO;HoW{_Z2fyt!8c@gX zrD2rUoCnx5FK7BqMF0eQ9E;XiSs_`uw`WjtG7^IaZI=`wd#0qpP~{=4dSk4Rd3HEt zqVczr#A8;6czyhGyMLbbVj+iqx)jDcYa|a1czVx_e?%J4)5GpfEMpm@)OBf3R$}I; zsvCxN8~%>Sk&A|-E-q>wDe8!N4+=2TxFH|GnT#2XHxnKVkLd8^X&?v+-LmgMw7jbZRP1mCc9ciExioG=d~fu;}8+kHqD`42=4k zdTyYYT?{+z{3ecLI{WzT{{UfT{9o`o%qiM8IC%sUCHXK)qEx7qIS(dCX+sC>m0jr4 zD)R%JAaXJw;K|94)yc~xp$~jVG>l_VP*8HhR~Qd+(`+RA;mg30q1$UZ&@{9RdzNb# zYk29{5UoIfO4<1zAPWmR*&w2!aC5DZldl>&pm0J)#Q|ERcsO$SR%nmahjaHsM#j4__$=R;Xozfi zasGF;UJXhsXq#`z>3gETHqhmyV(2IMUF(2i2l_gV(V#OlkR`~(lS;jJ<{ZvW?+5JU zhooNT9r1FW*QT@-m7)3+Blbv>PL5xHvundr!jjUwK%(_Q3{^~f6iBdecvccK?Y8-}7$R#C}(kbNSGa%INO&4+%k++d)8eq!P1Ykes&aufe6&4HEZ6>+) z$8Hig=qbq?!G#vO3K*lvsH}QfbfT#3uJd}#~7%zeaAR-mh>hB_L7HA79V|eB~ z2g5d>OChD1becF74kV4PsOfc?V_#e*QXL9N75X|MuD5A1H+9ri$2+WGASu4p)Z)+G zFu@ffLzM9fEr9d%MQ!V(yd9(Dr9AC_bMd<}DG!fP^eCESpgU<)$2hxOpk$GL*Wv{_@V`=tibnZ+t|IIO9QfilUL7cBd@vWw#5 zXy8rYoIUVxzL0{a*yjE5S3?t&?iEqG2=k-o62e-?2yCx~6^_FuYSxG4Jq8?b&P*O! zo4@^1t%z;g+a$Q->7IrJLWszZ=yX2tHOMryu;_`NtaV8*$d^Guox)6gIC(4PHz}^> zFAQt=3_Gv_oPrvx?V_`_&{iePrijDewuX79OTOvG05JPfhA?QocxoyZJ@V^xQCV}l zg8l|Hp4gY!ZdpGT!2cUQ7IvyQ`ZLh4Jgs9V#7&eNd(CCCKaQj5)8$)znf1mm6GIno z8a?1A0Bh@h3nQ^ooNi&$KGRR2sB^`3nqw{(ws!p7BrknAC8*LPW|JMYsgAn^W{J*1 zbC#2dbZn$_Cq`dA-qjtecZiV)iPefoPoI63P^;ew2DxpC`WdkrhDNw78|Re~YfF(U zcX?QX2UY9*;t-vVW`3!G3v0Zyb!!^Bn7LD7C^o|167Gu9>g%1_K3FAwKmqWi$va8Y zLN+ucVF(CvrUya|bmyt4)G@k>+$GfksvR`Uu$0cZn8JJ*i|8uH;PzPS2MXPX-_K}^ z-f$u?Y)G4R7&WN95QbpnkA40aiNFC79eJa&CcdrA1${bx)YVa~8+F16F=y}T0Om;9 z?qGL^IG`r$DYLOB1at&lT?yhTW;?Lt?C6={_Aqc(S_5+6N~SY^P-!&QpdNjIE`t9V zoS26taA}m}bH&yk0nb+LkWpe7Ph33#q!Q-=1yY&jHXslhKsQ&ziOB-`|x%d;`F0_r-6)>lGen|Bm5-mCRz^F!iR0v%P?-bIYXFXV;9OGG{ofF^Jm}-4N!!$A-~~DYgZYTiCLvA^`l(PNp{&a zw}6hd2e@XwR9+{5V*sr%)#VrMO)1&*fwNq^U1$96B1d%}<&=QaU&1cM4>=)?kWXlBo*X1ZkR};tQHOP1?i#}c z(S{>OF$yZ`8e3r58cqd1IG$AT0-Ck5?S>sg)%KQM9@a&3DZCTNaG8t6;ubZWD260? zRLw$TdI)GNs$zi=MX~1N8Sx|1D3`w|OR^1j@m1+ZY4spv6(WLKDF!P9LR9KVyqcJd zLD4qZ(V3P+6xZv*5@i^8j(2}AtUyC71Aj8n7#9FHr0H|^@77wr$>}O5mElJzcyT=i z&7n3o!7{%1Z%4>_+#Hv7r?9 zWBujL$(xLe4kD{H>X6ozHlHpdYSGfoP(+WdRc(Uw+BF3sq)oF^5p?^XkYr@&4Du0$ z51&yup;I4Z*c(@)jxne9lSe?6AO}?i?s$m2E4+m@`YYPeltbgt}5RY=R#{@NSfOniIp1WpZ(3FHx5dg_zzX z((5&1mnMX9;Jb5k##PiB{MQs@%8sZxMNG+}F#z^IQ1z|bFExLP;*YIeXLL0%=|RCa z1po3cJ8hHY(_2C^P5DWULnp=QcYVs<4`-%0mX*nYh)c*#O1mg+!TQU69r(Kf#B>9wE5`f1$CrC& zN@zk;t#};)$sU)n*XG0i?lmP*oeO?gpC$w5+LKuxrh*~|L4wD%=mywdmm*>AMugR( zJx-tK78`uBtrI5=)`i#YH*^`kJ`+Wh7KtY~6}Lxj3t+_N*HxOlmx|JH#67p8Eqe$V zgW>bSqAwNeb=-N0$MSrY{ ziyc&)QYgXcMvbywCuS#CMuSs~bshgBJ9?VoF2P~CgSi47C@#fqaPW{R=RcO8#DauH zO{7`h7A-0p<^aS#2jYkiRhA9^q7w-hGL!ei1cXl<;hcP^Q_lqz9#4+Y$Eup|NB(1g z|CdE6iEOAAAR}vo6CKV48hH+nv+r-vW=m;BneNfZ<{c02$$H1QaNTpTe)xEKv)TY@ zG^AM|Ftw1#b&?EctEDf(kibUPbr@hTw_nU(&{ux1fU^Jp0&6dn&|MsKsD;BO%7`2e z9SMX6Y*y)3U@(>I1VuJ5!a!g&tBbtGX_?X*pCQtGXH3D&XNHkc)2|rbWuX-tJy*&B zj2+^{zyumnS`4FA6q4yWU8O46F+)d$CvfI&O1$V};k3KT{(gd(GBk@451FM4wpTqO z5dmZ@eazInz?RC@2b_<;(3CM2@PO*R#jzWuR|{QNIa`15trG$HIwnXu4G@I|W*_`# zZoThG4p+uw%J%^ca5f3f&jJ^9!gxo7;f&F0g_X<|DNw47r8{}R02CY_N$vp!a_klLIa z_k(R4b8skc8-gfk*k5Fp>dSRyuWT1r-aUs;F(ya;h;<)$F0FaOm2`bz`T1e-pB(I)PqzB+29MH+6NgqR3p$fOf(tj8 zC_aH;o%YdSYaO9&Tgd#;R-hG$tPmaE5Mu8fj_)YHf0~Y@u`6L9ie|EbGkGvHKx=@d zEl+4&5Z%1s2MC)Ho#~84i`K?8t(dz^CsW zwt!H0eZ)Wb>^DzqkApRJ1?%TAkE0KUzhA@kufORzoL^G`ne>d51{*FJiA%)b@VHXe z6Kpd^BP3&n@i8zB(%Gm}|C{BrSC?~ON>u?Aj%*UBX)047f;E|_|gqDa*VRRh7(nbPdDT};s*B8CEE ziTQb^YOW~qu-f#RbQqsM(+Efx4IU4W=+B(#f*^)zTJ>~h>h+n4R|1_MXIbch<3ok^ z^Xyg1QBdPxcDkMc3GmhviXA5UlE5(I$`U?jKS#`UMj~ePskZplT(j^J49wY_V)jet z8QipdoAmKwn)5MJ?+1uE$Zvb8{PX~J<(EU<<$$kmSHXdT63`4C+lW|_fPuqYl|JGh z{DieSFbbYmdiZ32STsV=^t*r%cpMOb7qvr01Js65r_wwjs2CtkE|UV{klG`v7Qp^l z9oM7u#*uVwc5$=GVB-{EVIZ#WYjk42_rA1Xmq<%>e* zltaxBqjgp{$WK(QAPv`YZ=B91_10qH>ttpW>ZSIyBWd=B*Sdjyknl~G4uS3HomqDc zQHfBG6N%hA*9+TFr|mdBLzV!q3nOI>2Q}20-j*sGDr{P=@COT>u=_zOXcsWC)giH@ zsBbGA)t;y5Dy3aw;B~~KV`&=ufGAkisS4R0|~giyG4c#rJzVe+-iItBp9Y#P^tI6Nj3 z#t?Z)aXE>DglsmvJv?Egc!q|--GBsA#^c=7!G=9eTFsQrrRpDJi?>N`L7PNvrxrFkFf=`-{L2_RF*FW>0j@X8yNY$r5>XeaJc%*duHj(WxHYfn)l(|6L0lMwxWw+);JYe7+L{S>4k!#dN16U2f0du% zXZX4P9)Fyl<{{SwwzcR7S!Qy;1s+W1e~O>z#06+Oz-!XSqS<^yUMC>vRu_Y%Eb!6m z^`Heo;tarUx4xlN2ongkhAZJU4G`CXs@xE@Tq&cY)&#mzcNt__Riwcn?rTRDa?5IX z(JdEhGwT&Zm?@41Gh*+gO0l#T3a;{*yeJu1G1IKe*rkyiMB800J0A>J>NLFJ{(eWKo z)nE;$D$nq;u3;(~G#)hPX9XCw)G+D6+#_mnQy-J+rs(zkN@_`|DZ+O?@}*i=iORvl zwTn9qq~+;jhR#BD3~RoTKS0m zKX?2@i4s9Fs#?#Key74eQ{x}0^Uu`y_v(Cm^*$f^kIjD*f8R|71~7J?Dg94}{-?vg zQj)NFR0uAZzP{*wN!}nCV~${e>jsIa5p^%aQuw}2L1mvS zX_(2ApuI>=b1elc7@J&{>{f>@R&;vJDZb^1IJ7S`fa#c4NZPSFGrdZX$S9?pH%UC$ z$*jVQa42R5h)oc?+*zio`~`kSCbz=A0)>IySd@~15r7sUTLyC|)l?Up9ac*>9P>ok z00sstLqnBgsS&7VBr;uGxH#(q;^L0tP}X0g<~qE7cXPxPJ>UU`oyHE_JEW~un%o2h zWla0sBH`7lnD@F&TKm}E>}v0ti=CIiO3w>ey~>&g7+rN6ec+4M7<68dl{8rmy-=>K zE~`VS*_8=_7O!VIl{UpP>KzX%iMhpiYw@0hBjm%$QVqegp!1Kai9*rPcVKN$0@a~g zC)`4OS5Kw)zr;`$Wf>cbe8f{*1Dfn`H`j;opRd2EmdBv z3*iw(WEelOloL(rD&rzUvfv=Q7MCCvfW#i6vAWVxp+-l6n-$5)Te z?s)C4TQPuyO$!XvJcz>UvSe#fc|=y0-zg&Nzs?G`SFA6mum1oDCwH3urQhBDqsDxf z^#Mhf%l`lnyLX@bOWGFziBc2|4vhDxV}L|#Po=O0?%;gkKgWu@RiAMQ@m)Tb-+vI? zq0eNjDX+|^n#!=DJ5vlTKu=YI9aL;ODNTyJkyf(mQR6aMe@=D>iaKMW3^Q;AfoyMJ zkZR&CsJE5XoG&=79U{TVLkQS!Fv8Xd{PM5>ptWUGD=cM;0-zY2-ZmW#=_~eGIxv)0 zv8Sdb4M0Iz;f{>^lJx-)ufV1SUNUeFwxOa`5 zK)CTRf+Fywqf=E*kF5gd@=a$k3?}Jd<24nokxM|F1rSio!bMf8GLa4PVSsIzpG2rI zX_mt^(lJ0lEZ14FTFXCeXIAJErf`-Y>>N_Mz4e8Y_)r3^X^^$ZG9NcyF5{(%( zd;_p<=5sBj(=d+w_+VT?G~3{@yPrqQb$I;l=ZKF86(P-W4;l+jLalXY&p%&5Rd}!U z;{O2Ei%hKde4m;A&-;u20Au@){2%Qw1r?2K9V5HS6mbDSI1dei#L5k;GRK>e+4Lf% z4Ck0?d{c`F(Uyrj?Crf4TS~`xuAOx)aGqG;DL-#7EqXMh3kC;hHY*th#WpJ zL5{v)GG*xTLa`|opf?Hu5K-d?^K0MI2|vq;O95S3>KU_2b+>%^>mRH)N^TQfwo zXf&q9y{mRWSyf_>q91a9oQ7=33Mq@|xa>T1z0LX-c}1JyQI2p{jm)58WhtWaok@>W zkc_>tf@ZHRz$b;;Vh*R@FF?JWPtO3y@6bqC}2BV#V8AUGMJ1>c$RcCz3wCw z1E%)?EJW?&@!cf?$SHiMTAJ;yt4{qOpjX1u=se1hD3Em1=)>ImR1aMy9AAcK@t4+X z>>`&0Mrv4s(UF!oX@tF8$cC1OrVyHTEHQa_fZM!m4EyVdn-Ij|rQc0v2}!z9Qd7;a z<9IgpS9XPLPf0;jzR^N!8$ID&P>z*MX^R0l;mL<_ZXio&N4Gry69RlABtJgD!4(vU8fwu-4+v6Uvv#^!Gh{_z+(1ok?QemX64{(mfkNsksnS9y#YEe%wX&K)d6J9`GJWEiwTb|% zPVM6H=0!S|;c0-_*fX-2W?KtX#jf(;I$~7{N>#s>;J1ZzPN8B#R`U6H0!5QU8ah?W6 z@T3d4qLRlk0bLqS*rd^H033J_t||T@rqO^}GXq$zuv^G5R)w~WmM{%b!Bw_OLs-`^ z<>CiezUeQHB>`t-AlWx98i}j$42!T3-!4$`%Pf^sZt>5B)Fne>Q|T3Pa4;dz!J@gG zQG56kn%&13l##X|>L9awt#>8nprA_00@@QGLyxN-XWsqaUf&`yF4BPaYU?(F1bM_L zBzc+Om*ry=IY8VB+b>p&a5G}fZ7l>lEx1)P=+tPWQ=m~zy=%P6=&Hij>ypj6fgljc zzJ_}JklXV1yDpC5RMf+cRF{`*^@{;ST$R<%cw-eS*Uia-(VMDx)+0{kmW3j1NobH) z`=JYboWv$BZ~=R2k|Z&TF2_d(B4-Yo2BfU?AECc`=A)It|L(qm}|`(5{C8nxhX%a@BrZPcD45 z48)S&0frVRHp}7*0BxF~sCq`lRaDH%K&-KsJ}L?PK^Rq_Jg3I=e>SC>Xf+fy>*jPdkLF-PBVBLE9o1|J`{ndU)|OALJrU{27n{{G*qCl!rX@# znflqb8M4Z0l>Y#$`=9Oq0Jy+NAn1MKEg@i~=~3OduCUjNt$~euGI3Yn*7;lOx3v~g zok+LkTa;rrxVX|C;>k6RNHB#jFrlAwR)MSZnrMP;!l}#VeqxttcYK$>d9fQHw^CAM z9WHlQthin_Xmoo|BGwTom$C_ZFQgNEUl+GXIG~!WUoC(RX3iDViwt&zuB^!@x&*$n zQS%M+GXm`yEZ%00R9xk?_k=idKIltuLt;CW5sXk*n6a30c}CPsR9BXu)+|!E7YZ6a zzrG@lAFQ6T+c1yBP10=29j;T0G+IwtM$Z&c^wR}(VpHhmXxq^))o0FR%&3iGW*67N zq^A(EJ0QixxT<0@fhCMUD$sJrT}Q%s=t?*({mJE(4d$TqeWq7l@!}V;GbN`7ukkJN z2oi8yGL#Z@vi|^>8bYZm3xeLPy-wMpdSVTIA8h_SJ}alv_)e0f+$>qFEx>bwYNdeT ztoS=85DRL$Ag=;e2T1)tH`J9DELm*S0E96^uQ+T;q-xV z2Nq|+{{V1s4XWsd{6s7ck?Hk$04%qNHPCryPyqyVk^9fX>c_A3jowJIm~#wy;%+6d z*m**^4CEMqtz%f194%2^qf9tD(x=gxz)51MFDyZ5)Y&RGb1;S0;I$V}w}55ia^A38 z+brm1No=xUjl=nMJ^>as#G46a9f2%Z+$oXvhuIevunXH!w98Lv_6OJZN9JOo8^CU0 z8V}(kQgev>H?1`irVl78lIypm;VLQ~S4^wq3s0=?=1yZT0h6{eQ-g!3)Hbyrbp&Fhf5tf~-WpAdA3vN!Pb zihyU0esiJ_iD%=j;+3LQlHnj!S+!f5msD!NNrB009j6Zm@4tu*a*-v| z6&{*^@CX6|@Jq?K!$4t~XmnWDGOlG5()lR>sx8xqo@(qBn8#u1P!<5DO?m+Ua4#~b zzZ6`ShDN%0>p1ddsDeX}Au{PBMoPP#GBho*!RK?pj=pcl=v_?QDo>)p7;sD1Md6d1POMq|kAxc4P(vxE=c0!!)% zFB-lJ1QDgw69wDM9Bu_}0`6Z%4aCwmUP2Lko}&K5-lJ3xzGZR0r|&SRIm)?yA<|MM zWgtiq1Y6p0_laR^pWKQFlGJtPLjj8@U)~}X!FeC<2|?3~-r{uH z_@X043kfrZs7-z1)St2Yjw-DRkFg1W%{sl}GA*>-uJ;PUQ$bY*w83u?CTd(+om9#w z=v9uYC@pSM76%>TmoBTSiliCFV%iY>{{Y2}i4v~MZ^{m|Fsh+34h`nD;Z2wgF2Fq( zev+83?^LUyi{==NyHl9tO}$|4#Gml!Zue=P*H@fDcQT{6DkU2D+TjN@ty5$^&|t?F8k>!;X@62{T6)ew9e3#!Um>F(S`AnE1`d+b z9?+~h>e2LJES&;S1>=S*>2N{~QX~18Y$rCL)iaVl^0ai_K^mZ5`Dw6KQp9NWr{#{5 z8EPxqRFKn({h;fPrP)246o4+O8i@)fM|qE7(h44;TDsG$BGJ^Su8|d^X?J*wrwiUV zN)=jX+8j<$jIbz|2bpw7>Og2a~0P;r!E{A7$(yX*N z++A)LCG~)go<`x+RCzZTfNvQ!1IXrymc!~xDN*90Y=tFm?j(R2h;Y7=t{$-K&}4hR zi|0B|h3J~!0y|EvgUQ@!^q2eQd5$^24+d;?7vCuvk@9 z)-vGWpBMc@dp0C)nmioM2P#pddc^vhTzM8zJ`MVxKK)ORey7X7Q{w*ssq;V7`CsZ7 z)j{8|h9L{ak~<_KecRw)(?nhgg{_@fW%&!@s;5tPkMOxI@O*v)Tp|507e0Pbsfya*2VPCF zz@=vlOXq~fQG^2#ZXBE9_fZlc#X84*<;G?}5-V`Wc~k+OI>kD%$UxqB#j>Ty>|lk3 zzj?)fxA6xI&@0rs%rpe7`70ULA>lb*?BWb_EI zc(V_KTtNzU2Q_EEh5rBpXtuwE$VXyS`6aAID)WmjMeq=BJx_BXpUH%1q>E+ zu(qB$!mcBD+_e#Ikqw9otw)wi&R{Bon~9StWw8JavdT4&+$(X|TuoIeDRvs%vN&PE zV@nhknM*->?TuyZH57ncH!2Hd>A9p=km;!Z08>4W$a8TmIxerPJ0cj(An~2TQw)?c z!HOJoo5iC=%Y)3cnpS%qA&eR80ml>V53SR8KQmR0HdPfCRZzZ3SY^snx^8aMf!Y56 zB4BVS<{trQ!eUYnT#tCRh4?owcd)q#eE#C&ixnxC``ytO7G=0$x!O6?970p3p zl;BaZ0uf515yc9wv9*t)Dzu`^!G%M3+%?406p%YAP=K@?%I{EPchXQN37vP2i1xZ_ zbIGj0C5qMDvX)QG#bJi>H<+rg&y*NN$?AU>&UA~;HCb9aPn>?I&OcM#ar&P){ZE{Jr_Mi9=O3wAhpe5N#5YG(mwht&&$#}4P)bW48MuDv+u9GA_DkX^ z6kXO$6PIPrwtpYU-aot&4l-C2@WRquo`Qjlu2?Al0KBHODClX$^Ds=wdlz8Y5qMk% zVN$UB!yB<|0@1XwGnW$xs6;4hI63|l0ws?T{{WP4Hd;G-Py>(M<*_p7VT3xMW_#6sq>-iHxdaKTV(z-j4%^+kr$?$>XvFgJ8&?(=_-@UiJj zP~2(-w$Pa8!2wq3aK}8;UD%2^v2CC7CWQm9FepL3PpoOX!tTek0-zir1g%uvx}-ze z3b{rOI;U|F^6I4PZ&m<6V$w;-_Jpq%Iu7KchX7?V1~js<14eY~{fH>4`%7TPLNmY% z8Pvw*w|va}z&&Agig}_!2P`QgE9yq(uR~kw5c1rY5JZ#|Bwzr2NuNQB+_JeMsRpyWk#vT0K{5pngemmU?dLdAeC4&s&SJz-&SpSh zTORg7jsmDk8!lr0g)wk4oYkxbp+T|CqIiyozet!s=wDI@3-w1%B>_+5r+Gx!UF4qR zqkZc?#vWgd17&lN>oy=yYlS{Z6rfu+5xz6LNwSk7XivR zE3VO8VQGY;Kg3t zVht>rcFTz6?$M&1$0S1FZh{E9`Jtp;1bUNX9jpc)X;6_(#3&^0fu+HN2eiH9O%Y`U zcLJMj3hwUE*n28sK$M8Afj?38k>j-pVpdOG73D}*Yc8S_f z(sq+>JjwU!46-iyVvVZ#BHLd?4_JF3U)F)BFH8?d>4CNfrW(QMf#`h@4F{y{J*R0q zMC}u_ouup{>GRu|zx7R`dPR3PHDZD&3acIu@mI(nP^aw5rfS6zO6jWpZ@B(^Z|!}F zSzkwj02CB>rFg3vz-_B&3hNPyvMA?TuAWx|EvjKQwCY`Ju_e1kikj^fx|`cITurKV zp1aeDkDMOBtT;JPEEu-jy3VEMg)nssjh^x5JlP<&T{voHFcm9cp{uKn#acmkn+FUN z5E~B6vmJg8ewT8H&IQD5al3)WM&R{I<`Pz+RO@CVIis3Qu5}p|Ga+b!#oYNqp%!{W zjjHQ&efB5J3={!Ttmw-QAcrbIJ171SpmiDpY;exz4a9M0@Q(dKA*^1Lv`)nBCuuuL z+D_6th4h52s_jrGc1IBu=Bm;b>-NJ zF1p)8e+?_7P-rsX8t(U3dWP&QQ(y(_aqp`rW}!|h-eD{*%3#W?zFrAo?7L=*qOkFp zXit|)Rqnq30Qh;NeM=2&!#ptSyea^oJwN;sr1T(+2cxVF-4(gVTIgDKni>#_dB7p9 zA#bE^Rkct>g3z*^ku;U0E%g5Y(rAo;u{tqQK@F8!IBmuwoK_W%jZd}z03(^BZ74Cv!w_C28}ELallw)XqVxlOO7*4{0YwlkUbOp^#_{OOk& zBdz+Og@{aVq$>%N5cB6eB81puthDJ-dx~i9T*b;t&5L1f2*=A|s#5Db#h|OVe4B9h z%rExgg zb~gG!?v$%;`myqUm8(f?P#8P1_PTZx7mQzjqe@d7Pd}=P!ZNIA`>{{e8Ye>c^qDtM z;x>n)3ey*we8SV#TdFSI4L@_kCeRk{3*MjoKWN897$Zy>R1F&t8Ze^0V9K&?HFaIx z51hzYqlLinl8&BHE(2sM3)P_xx`^>2_v4F&W({-=8c|>KQLwPl9Tu}2{{YlYp|O1- zlin1ukR~dW**;DBn@Gb@R(anu=ijL5vji$`-R^wb^#s;10^OV0#QAsXD|kw;6@{tu zZ`8dd6;)g1t;bdBfEU14rDF&p3U_O9()G`ibQOXwMQ(>CLFXHfr?K0G7$C9)ysWIF zF;U@aRmcs+=o#%+jFF_dbu)H`*k)*5sBV(7M>cI+mbh74ZASkEp8~# zwQC!otP$e*baFNhoNhNx?`pJQh;LS{Wn~bVNM0%{0Eq{w+JzY+_|D04wud>~04(vK z-6dr0DlNzg%8LC<-acvagb~q)p<=^+9UDI)l}`=I zaHhD^K41Y zq;#lj1yEmBSvUQ1*I_Dh7N|>wYoXkyl#&R|SPn;C5ROv*ge~WEr!8UzB)`cpJpmNp~j`%0rH^wNRl->D2Q3YH8V{Ef$o6)Jp= zOUJ1`?fL%z(@PQNTw3iL2MyVyP`bK7VvBCvb&UI@KpVT!+bzP;3Y%Ln{v%;BL?{!q ze^D_hGTOWId$sG@X7zIJY(I%k#14Q_{{ZP7p2=8(^zMrjWy&e{@y-(S_Dj*nsx-V{ z{cb+EvM?Dt0q9GXk@QG87PMXx`dRHMtd1)zEnsD==ApGSGC@m*yyI_0wHEd} zH5LHw2S5t#(m8R7LLeSY-Mv`qA(kVFMqY1H+-Yo`Yd-ZCYQkvJp^hu1L|ipp3J^F8 z#2861)%lh@yC7&6_KQ~4x>ByZpbjuufCcWXv$@_DJsUU*Rf9&jfo<)v91ow^gO5B* zuCuIhElZf?A$4F2eOX7fJPz&VucT%O3$^q`;>x Date: Mon, 26 Jun 2023 18:17:24 +0300 Subject: [PATCH 19/25] Update README.md --- README.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/README.md b/README.md index c795afe6..1e610593 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,25 @@ +# Cycle-CenterNet +Table structure parsing (TSP), Wired Table in the Wild (WTW). + +The unofficial Cycle-CenterNet architecture repository, based on the MMDetection fork. +![](demo/schema.jpg) +> [**Parsing table structures in the wild**]([http://openaccess.thecvf.com/content/ICCV2021/papers/Long_Parsing_Table_Structures_in_the_Wild_ICCV_2021_paper.pdf]), +> Long, R., Wang, W., Xue, N., Gao, F., Yang, Z., Wang, Y., & Xia, G. S. + +## Abstract +This paper tackles the problem of table structure parsing (TSP) from images in the wild. In contrast to existing studies that mainly focus on parsing well-aligned tabular images with simple layouts from scanned PDF documents, we aim to establish a practical table structure parsing system for real-world scenarios where tabular input images are taken or scanned with severe deformation, bending or occlusions. For designing such a system, we propose an approach named Cycle-CenterNet on the top of CenterNet with a novel cycle-pairing module to simultaneously detect and group tabular cells into structured tables. In the cycle-pairing module, a new pairing loss function is proposed for the network training. Alongside with our Cycle-CenterNet, we also present a large-scale dataset, named Wired Table in the Wild (WTW), which includes well-annotated structure parsing of multiple style tables in several scenes like photo, scanning files, web pages, etc.. In experiments, we demonstrate that our Cycle-CenterNet consistently achieves the best accuracy of table structure parsing on the new WTW dataset by 24.6% absolute improvement evaluated by the TEDS metric. A more comprehensive experimental analysis also validates the advantages of our proposed methods for the TSP task. + +Contact: [arkhipov.ai@phystech.edu](mailto:arkhipov.ai@phystech.edu). Any questions or discussions are welcomed! + +## Models + +## Results + +## Preprocessing + +## Post-processing + +# MMDetection README
 
From 4ab0be4ce7386782be15925e5b86950227179764 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=85=D0=B8=D0=BF=D0=BE=D0=B2=20=D0=90=D0=BB?= =?UTF-8?q?=D0=B5=D0=BA=D1=81=D0=B0=D0=BD=D0=B4=D1=80?= <32227207+ArchieAlexArkhipov@users.noreply.github.com> Date: Mon, 26 Jun 2023 18:48:27 +0300 Subject: [PATCH 20/25] add models to README.md --- README.md | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 1e610593..89ff32fc 100644 --- a/README.md +++ b/README.md @@ -5,14 +5,23 @@ The unofficial Cycle-CenterNet architecture repository, based on the MMDetection ![](demo/schema.jpg) > [**Parsing table structures in the wild**]([http://openaccess.thecvf.com/content/ICCV2021/papers/Long_Parsing_Table_Structures_in_the_Wild_ICCV_2021_paper.pdf]), > Long, R., Wang, W., Xue, N., Gao, F., Yang, Z., Wang, Y., & Xia, G. S. - -## Abstract -This paper tackles the problem of table structure parsing (TSP) from images in the wild. In contrast to existing studies that mainly focus on parsing well-aligned tabular images with simple layouts from scanned PDF documents, we aim to establish a practical table structure parsing system for real-world scenarios where tabular input images are taken or scanned with severe deformation, bending or occlusions. For designing such a system, we propose an approach named Cycle-CenterNet on the top of CenterNet with a novel cycle-pairing module to simultaneously detect and group tabular cells into structured tables. In the cycle-pairing module, a new pairing loss function is proposed for the network training. Alongside with our Cycle-CenterNet, we also present a large-scale dataset, named Wired Table in the Wild (WTW), which includes well-annotated structure parsing of multiple style tables in several scenes like photo, scanning files, web pages, etc.. In experiments, we demonstrate that our Cycle-CenterNet consistently achieves the best accuracy of table structure parsing on the new WTW dataset by 24.6% absolute improvement evaluated by the TEDS metric. A more comprehensive experimental analysis also validates the advantages of our proposed methods for the TSP task. Contact: [arkhipov.ai@phystech.edu](mailto:arkhipov.ai@phystech.edu). Any questions or discussions are welcomed! +## Abstract +This paper tackles the problem of table structure parsing (TSP) from images in the wild. In contrast to existing studies that mainly focus on parsing well-aligned tabular images with simple layouts from scanned PDF documents, we aim to establish a practical table structure parsing system for real-world scenarios where tabular input images are taken or scanned with severe deformation, bending or occlusions. For designing such a system, we propose an approach named Cycle-CenterNet on the top of CenterNet with a novel cycle-pairing module to simultaneously detect and group tabular cells into structured tables. In the cycle-pairing module, a new pairing loss function is proposed for the network training. Alongside with our Cycle-CenterNet, we also present a large-scale dataset, named Wired Table in the Wild (WTW), which includes well-annotated structure parsing of multiple style tables in several scenes like photo, scanning files, web pages, etc.. In experiments, we demonstrate that our Cycle-CenterNet consistently achieves the best accuracy of table structure parsing on the new WTW dataset by 24.6% absolute improvement evaluated by the TEDS metric. A more comprehensive experimental analysis also validates the advantages of our proposed methods for the TSP task. + ## Models +[**Link**](https://drive.google.com/file/d/1ZQZycYwWXMlfZnvC9hFUCtlqd4eOwP8v/view?usp=share_link) to download CenterNet(ResNet backbone) + +[**Link**](https://drive.google.com/file/d/1aZ2IF0tQq1Ino4QAsmrk8jcJ85urqsFE/view?usp=share_link) to download CenterNet(DLA backbone) + +[**Link**](https://drive.google.com/file/d/1oeqtA84eF_KJi953f8qX8cggDBjOVS3A/view?usp=share_link) to download Cycle-CenterNet(DLA backbone) trained on bounding boxes + +[**Link**](https://drive.google.com/file/d/1taFK_co-9ofbL1pivhKrqHf8-icb1ZaK/view?usp=share_link) **to download Cycle-CenterNet(DLA backbone) trained on bounding quadrangles** + + ## Results ## Preprocessing From 198f75246423aba3a680cf4099132080b476a669 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=85=D0=B8=D0=BF=D0=BE=D0=B2=20=D0=90=D0=BB?= =?UTF-8?q?=D0=B5=D0=BA=D1=81=D0=B0=D0=BD=D0=B4=D1=80?= <32227207+ArchieAlexArkhipov@users.noreply.github.com> Date: Mon, 26 Jun 2023 19:49:59 +0300 Subject: [PATCH 21/25] configs --- ...x_cycle_centernet_dla34_dcnv2_150e_coco.py | 326 +++++++++++++++++ .../wtw_centernet_dla34_dcnv2_150e_coco.py | 302 ++++++++++++++++ .../wtw_centernet_resnet34_dcnv2_150e_coco.py | 300 ++++++++++++++++ ...d_cycle_centernet_dla34_dcnv2_150e_coco.py | 327 ++++++++++++++++++ 4 files changed, 1255 insertions(+) create mode 100644 configs/centernet/wtw_bbox_cycle_centernet_dla34_dcnv2_150e_coco.py create mode 100644 configs/centernet/wtw_centernet_dla34_dcnv2_150e_coco.py create mode 100644 configs/centernet/wtw_centernet_resnet34_dcnv2_150e_coco.py create mode 100644 configs/centernet/wtw_quad_cycle_centernet_dla34_dcnv2_150e_coco.py diff --git a/configs/centernet/wtw_bbox_cycle_centernet_dla34_dcnv2_150e_coco.py b/configs/centernet/wtw_bbox_cycle_centernet_dla34_dcnv2_150e_coco.py new file mode 100644 index 00000000..5188607a --- /dev/null +++ b/configs/centernet/wtw_bbox_cycle_centernet_dla34_dcnv2_150e_coco.py @@ -0,0 +1,326 @@ +TEST_NAME = "31_long_check" +EVAL_LAG = 30 +CHECKPOINT_LAG = 5 +EPOCHS = 150 +LR = 0.00125 +BACKBONE = "DLANetMMDet3D" +BATCH = 8 +local_maximum_kernel = 1 +LOG_LAG = 100 +TAGS = [ + f"local_maximum_kernel={local_maximum_kernel}", + "1hm", + f"{EPOCHS}_epoch", + f"{LR}_lr", + f"{EVAL_LAG}_eval_lag", + f"{CHECKPOINT_LAG}_chkpt_lag", + f"{BACKBONE}_backbone", + f"{BATCH}_batch", +] +# DATA AND AUG +dataset_type = "CocoDataset" +data_root = "/home/aiarhipov/datasets/WTW-dataset/" + +img_norm_cfg = dict(mean=[103.53, 116.28, 123.675], std=[1.0, 1.0, 1.0], to_rgb=False) + +data = dict( + samples_per_gpu=BATCH, + workers_per_gpu=2, + train=dict( + type=dataset_type, + ann_file="/home/aiarhipov/datasets/WTW-dataset/train/train.json", + img_prefix="train/images/", + pipeline=[ + dict(type="LoadImageFromFile", to_float32=True, color_type="color"), + dict(type="LoadAnnotations", with_bbox=True), + dict( + type="PhotoMetricDistortion", + brightness_delta=32, + contrast_range=(0.5, 1.5), + saturation_range=(0.5, 1.5), + hue_delta=18, + ), + dict( + type="RandomCenterCropPad", + crop_size=(1024, 1024), + ratios=(0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3), + mean=[0, 0, 0], + std=[1, 1, 1], + to_rgb=True, + test_pad_mode=None, + ), + dict(type="Resize", img_scale=(1024, 1024), keep_ratio=True), + dict(type="RandomFlip", flip_ratio=0.5), + dict( + type="Normalize", + mean=[103.53, 116.28, 123.675], + std=[1.0, 1.0, 1.0], + to_rgb=False, + ), + dict(type="DefaultFormatBundle"), + dict(type="Collect", keys=["img", "gt_bboxes", "gt_labels"]), + ], + data_root="/home/aiarhipov/datasets/WTW-dataset/", + classes=("box",), + ), + val_loss=dict( + type=dataset_type, + ann_file="/home/aiarhipov/datasets/WTW-dataset/test/test.json", + img_prefix="test/images/", + pipeline=[ + dict(type="LoadImageFromFile", to_float32=True, color_type="color"), + dict(type="LoadAnnotations", with_bbox=True), + dict( + type="PhotoMetricDistortion", + brightness_delta=32, + contrast_range=(0.5, 1.5), + saturation_range=(0.5, 1.5), + hue_delta=18, + ), + dict( + type="RandomCenterCropPad", + crop_size=(1024, 1024), + ratios=(0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3), + mean=[0, 0, 0], + std=[1, 1, 1], + to_rgb=True, + test_pad_mode=None, + ), + dict(type="Resize", img_scale=(1024, 1024), keep_ratio=True), + dict(type="RandomFlip", flip_ratio=0.5), + dict( + type="Normalize", + mean=[103.53, 116.28, 123.675], + std=[1.0, 1.0, 1.0], + to_rgb=False, + ), + dict(type="DefaultFormatBundle"), + dict(type="Collect", keys=["img", "gt_bboxes", "gt_labels"]), + ], + data_root="/home/aiarhipov/datasets/WTW-dataset/", + classes=("box",), + ), + val=dict( + type=dataset_type, + ann_file="/home/aiarhipov/datasets/WTW-dataset/test/test.json", + img_prefix="test/images/", + pipeline=[ + dict(type="LoadImageFromFile", to_float32=True), + dict( + type="MultiScaleFlipAug", + scale_factor=1.0, + flip=False, + transforms=[ + dict(type="Resize", keep_ratio=True), + dict( + type="RandomCenterCropPad", + ratios=None, + border=None, + mean=[0, 0, 0], + std=[1, 1, 1], + to_rgb=True, + test_mode=True, + test_pad_mode=["logical_or", 31], + test_pad_add_pix=1, + ), + dict(type="RandomFlip"), + dict( + type="Normalize", + mean=[103.53, 116.28, 123.675], + std=[1.0, 1.0, 1.0], + to_rgb=False, + ), + dict(type="DefaultFormatBundle"), + dict( + type="Collect", + meta_keys=( + "filename", + "ori_filename", + "ori_shape", + "img_shape", + "pad_shape", + "scale_factor", + "flip", + "flip_direction", + "img_norm_cfg", + "border", + ), + keys=["img"], + ), + ], + ), + ], + data_root="/home/aiarhipov/datasets/WTW-dataset/", + classes=("box",), + ), + test=dict( + type=dataset_type, + ann_file="/home/aiarhipov/datasets/WTW-dataset/test/test.json", + img_prefix="test/images/", + pipeline=[ + dict(type="LoadImageFromFile", to_float32=True), + dict( + type="MultiScaleFlipAug", + scale_factor=1.0, + flip=False, + transforms=[ + dict(type="Resize", keep_ratio=True), + dict( + type="RandomCenterCropPad", + ratios=None, + border=None, + mean=[0, 0, 0], + std=[1, 1, 1], + to_rgb=True, + test_mode=True, + test_pad_mode=["logical_or", 31], + test_pad_add_pix=1, + ), + dict(type="RandomFlip"), + dict( + type="Normalize", + mean=[103.53, 116.28, 123.675], + std=[1.0, 1.0, 1.0], + to_rgb=False, + ), + dict(type="DefaultFormatBundle"), + dict( + type="Collect", + meta_keys=( + "filename", + "ori_filename", + "ori_shape", + "img_shape", + "pad_shape", + "scale_factor", + "flip", + "flip_direction", + "img_norm_cfg", + "border", + ), + keys=["img"], + ), + ], + ), + ], + data_root="/home/aiarhipov/datasets/WTW-dataset/", + classes=("box",), + ), +) + + +# MODEL CycleCenterNet(dcnv2) DLANetMMDet3D, +load_from = None +resume_from = None + +model = dict( + type="CenterNet", + backbone=dict( + type=BACKBONE, + depth=34, + norm_cfg=dict(type="BN"), + init_cfg=dict( + type="Pretrained", + checkpoint="http://dl.yf.io/dla/models/imagenet/dla34%2Btricks-24a49e58.pth", + ), + ), + neck=dict( + type="CTResNetNeck", + in_channel=512, + num_deconv_filters=(256, 128, 64), + num_deconv_kernels=(4, 4, 4), + use_dcn=True, + ), + bbox_head=dict( + type="CycleCenterNetHead", + in_channel=64, + feat_channel=64, + loss_center_heatmap=dict(type="GaussianFocalLoss", loss_weight=1.0), + loss_offset=dict(type="L1Loss", loss_weight=1.0), + loss_c2v=dict(type="L1Loss", loss_weight=1.0), + loss_v2c=dict(type="L1Loss", loss_weight=0.5), + ), + train_cfg=None, + test_cfg=dict( + topk=3000, + local_maximum_kernel=local_maximum_kernel, + max_per_img=3000, + # nms=dict(type="nms", iou_threshold=0.475, split_thr=2000), + ), +) + + +# GPU +gpu_ids = [5] +device = "cuda" + + +# OPTIMIZATION +optimizer = dict(type="SGD", lr=LR, momentum=0.9, weight_decay=0.0001) +# Based on the default settings of modern detectors, the SGD effect is better +# than the Adam in the source code, so we use SGD default settings and +# if you use adam+lr5e-4, the map is 29.1. +optimizer_config = dict(grad_clip=dict(max_norm=35, norm_type=2)) + + +# LEARNING POLICY +runner = dict(type="EpochBasedRunner", max_epochs=EPOCHS) # the real epoch is 28*5=140 + +# Based on the default settings of modern detectors, we added warmup settings. +lr_config = dict( + policy="step", + warmup="linear", + warmup_iters=1000, + warmup_ratio=0.001, + step=[90, 120], # the real step is [18*5, 24*5] +) + +# NOTE: `auto_scale_lr` is for automatically scaling LR, +# USER SHOULD NOT CHANGE ITS VALUES. +# base_batch_size = (8 GPUs) x (16 samples per GPU) +auto_scale_lr = dict(enable=False, base_batch_size=16) + + +# LOGGING +work_dir = f"/home/aiarhipov/centernet/exps/{TEST_NAME}" +INTERVAL = int(10976 / (LOG_LAG * BATCH)) + (10976 % (LOG_LAG * BATCH) > 0) +log_config = dict( + interval=INTERVAL, + hooks=[ + dict(type="TextLoggerHook"), + dict(type="TensorboardLoggerHook"), + dict( + type="MMDetWandbHook", + init_kwargs={ + "project": "CenterNet", + "entity": "centernet", + "name": TEST_NAME, + "dir": "/home/aiarhipov/centernet/exps/wandb", + "tags": TAGS, + }, + interval=INTERVAL, + log_checkpoint=True, + log_checkpoint_metadata=False, + num_eval_images=5, + ), + ], +) +log_level = "INFO" + + +# EVALUATION +evaluation = dict(interval=EVAL_LAG, metric="bbox") +checkpoint_config = dict(interval=CHECKPOINT_LAG, max_keep_ckpts=5) + + +# RUNTIME +seed = 0 + +custom_hooks = [dict(type="NumClassCheckHook")] +dist_params = dict(backend="nccl") +workflow = [("train", 1), ("val", 1)] + +# disable opencv multithreading to avoid system being overloaded +opencv_num_threads = 0 +# set multi-process start method as `fork` to speed up the training +mp_start_method = "fork" diff --git a/configs/centernet/wtw_centernet_dla34_dcnv2_150e_coco.py b/configs/centernet/wtw_centernet_dla34_dcnv2_150e_coco.py new file mode 100644 index 00000000..4798d795 --- /dev/null +++ b/configs/centernet/wtw_centernet_dla34_dcnv2_150e_coco.py @@ -0,0 +1,302 @@ +TEST_NAME = "16_paper_params_dla34_batch8" + +# DATA AND AUG +dataset_type = "CocoDataset" +data_root = "/home/aiarhipov/datasets/WTW-dataset/" + +img_norm_cfg = dict(mean=[103.53, 116.28, 123.675], std=[1.0, 1.0, 1.0], to_rgb=False) + +data = dict( + samples_per_gpu=8, + workers_per_gpu=2, + train=dict( + type=dataset_type, + ann_file="/home/aiarhipov/datasets/WTW-dataset/train/train.json", + img_prefix="train/images/", + pipeline=[ + dict(type="LoadImageFromFile", to_float32=True, color_type="color"), + dict(type="LoadAnnotations", with_bbox=True), + dict( + type="PhotoMetricDistortion", + brightness_delta=32, + contrast_range=(0.5, 1.5), + saturation_range=(0.5, 1.5), + hue_delta=18, + ), + dict( + type="RandomCenterCropPad", + crop_size=(1024, 1024), + ratios=(0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3), + mean=[0, 0, 0], + std=[1, 1, 1], + to_rgb=True, + test_pad_mode=None, + ), + dict(type="Resize", img_scale=(1024, 1024), keep_ratio=True), + dict(type="RandomFlip", flip_ratio=0.5), + dict( + type="Normalize", + mean=[103.53, 116.28, 123.675], + std=[1.0, 1.0, 1.0], + to_rgb=False, + ), + dict(type="DefaultFormatBundle"), + dict(type="Collect", keys=["img", "gt_bboxes", "gt_labels"]), + ], + data_root="/home/aiarhipov/datasets/WTW-dataset/", + classes=("box",), + ), + val_loss=dict( + type=dataset_type, + ann_file="/home/aiarhipov/datasets/WTW-dataset/test/test.json", + img_prefix="test/images/", + pipeline=[ + dict(type="LoadImageFromFile", to_float32=True, color_type="color"), + dict(type="LoadAnnotations", with_bbox=True), + dict( + type="PhotoMetricDistortion", + brightness_delta=32, + contrast_range=(0.5, 1.5), + saturation_range=(0.5, 1.5), + hue_delta=18, + ), + dict( + type="RandomCenterCropPad", + crop_size=(1024, 1024), + ratios=(0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3), + mean=[0, 0, 0], + std=[1, 1, 1], + to_rgb=True, + test_pad_mode=None, + ), + dict(type="Resize", img_scale=(1024, 1024), keep_ratio=True), + dict(type="RandomFlip", flip_ratio=0.5), + dict( + type="Normalize", + mean=[103.53, 116.28, 123.675], + std=[1.0, 1.0, 1.0], + to_rgb=False, + ), + dict(type="DefaultFormatBundle"), + dict(type="Collect", keys=["img", "gt_bboxes", "gt_labels"]), + ], + data_root="/home/aiarhipov/datasets/WTW-dataset/", + classes=("box",), + ), + val=dict( + type=dataset_type, + ann_file="/home/aiarhipov/datasets/WTW-dataset/test/test.json", + img_prefix="test/images/", + pipeline=[ + dict(type="LoadImageFromFile", to_float32=True), + dict( + type="MultiScaleFlipAug", + scale_factor=1.0, + flip=False, + transforms=[ + dict(type="Resize", keep_ratio=True), + dict( + type="RandomCenterCropPad", + ratios=None, + border=None, + mean=[0, 0, 0], + std=[1, 1, 1], + to_rgb=True, + test_mode=True, + test_pad_mode=["logical_or", 31], + test_pad_add_pix=1, + ), + dict(type="RandomFlip"), + dict( + type="Normalize", + mean=[103.53, 116.28, 123.675], + std=[1.0, 1.0, 1.0], + to_rgb=False, + ), + dict(type="DefaultFormatBundle"), + dict( + type="Collect", + meta_keys=( + "filename", + "ori_filename", + "ori_shape", + "img_shape", + "pad_shape", + "scale_factor", + "flip", + "flip_direction", + "img_norm_cfg", + "border", + ), + keys=["img"], + ), + ], + ), + ], + data_root="/home/aiarhipov/datasets/WTW-dataset/", + classes=("box",), + ), + test=dict( + type=dataset_type, + ann_file="/home/aiarhipov/datasets/WTW-dataset/test/test.json", + img_prefix="test/images/", + pipeline=[ + dict(type="LoadImageFromFile", to_float32=True), + dict( + type="MultiScaleFlipAug", + scale_factor=1.0, + flip=False, + transforms=[ + dict(type="Resize", keep_ratio=True), + dict( + type="RandomCenterCropPad", + ratios=None, + border=None, + mean=[0, 0, 0], + std=[1, 1, 1], + to_rgb=True, + test_mode=True, + test_pad_mode=["logical_or", 31], + test_pad_add_pix=1, + ), + dict(type="RandomFlip"), + dict( + type="Normalize", + mean=[103.53, 116.28, 123.675], + std=[1.0, 1.0, 1.0], + to_rgb=False, + ), + dict(type="DefaultFormatBundle"), + dict( + type="Collect", + meta_keys=( + "filename", + "ori_filename", + "ori_shape", + "img_shape", + "pad_shape", + "scale_factor", + "flip", + "flip_direction", + "img_norm_cfg", + "border", + ), + keys=["img"], + ), + ], + ), + ], + data_root="/home/aiarhipov/datasets/WTW-dataset/", + classes=("box",), + ), +) + + +# MODEL CenterNet(dcnv2) without changes +load_from = None +resume_from = None + +model = dict( + type="CenterNet", + backbone=dict( + type="DLANetMMDet3D", + depth=34, + norm_cfg=dict(type="BN"), + init_cfg=dict( + type="Pretrained", + checkpoint="http://dl.yf.io/dla/models/imagenet/dla34%2Btricks-24a49e58.pth", + ), + ), + neck=dict( + type="CTResNetNeck", + in_channel=512, + num_deconv_filters=(256, 128, 64), + num_deconv_kernels=(4, 4, 4), + use_dcn=True, + ), + bbox_head=dict( + type="CenterNetHead", + num_classes=1, + in_channel=64, + feat_channel=64, + loss_center_heatmap=dict(type="GaussianFocalLoss", loss_weight=1.0), + loss_wh=dict(type="L1Loss", loss_weight=0.1), + loss_offset=dict(type="L1Loss", loss_weight=1.0), + ), + train_cfg=None, + test_cfg=dict(topk=3000, local_maximum_kernel=1, max_per_img=3000), +) + + +# GPU +gpu_ids = [6] +device = "cuda" + + +# OPTIMIZATION +optimizer = dict(type="SGD", lr=0.00125, momentum=0.9, weight_decay=0.0001) +# Based on the default settings of modern detectors, the SGD effect is better +# than the Adam in the source code, so we use SGD default settings and +# if you use adam+lr5e-4, the map is 29.1. +optimizer_config = dict(grad_clip=dict(max_norm=35, norm_type=2)) + + +# LEARNING POLICY +runner = dict(type="EpochBasedRunner", max_epochs=150) # the real epoch is 28*5=140 + +# Based on the default settings of modern detectors, we added warmup settings. +lr_config = dict( + policy="step", + warmup="linear", + warmup_iters=1000, + warmup_ratio=0.001, + step=[90, 120], # the real step is [18*5, 24*5] +) + +# NOTE: `auto_scale_lr` is for automatically scaling LR, +# USER SHOULD NOT CHANGE ITS VALUES. +# base_batch_size = (8 GPUs) x (16 samples per GPU) +auto_scale_lr = dict(enable=False, base_batch_size=16) + + +# LOGGING +work_dir = f"/home/aiarhipov/centernet/exps/{TEST_NAME}" + +log_config = dict( + interval=1000, + hooks=[ + dict(type="TextLoggerHook"), + dict(type="TensorboardLoggerHook"), + dict( + type="MMDetWandbHook", + init_kwargs={ + "project": "CenterNet", + "entity": "centernet", + "name": TEST_NAME, + }, + interval=1000, + log_checkpoint=True, + log_checkpoint_metadata=True, + num_eval_images=15, + ), + ], +) +log_level = "INFO" + + +# EVALUATION +evaluation = dict(interval=30, metric="bbox") +checkpoint_config = dict(interval=30) + + +# RUNTIME +seed = 0 + +custom_hooks = [dict(type="NumClassCheckHook")] +dist_params = dict(backend="nccl") +workflow = [("train", 1), ("val", 1)] + +# disable opencv multithreading to avoid system being overloaded +opencv_num_threads = 0 +# set multi-process start method as `fork` to speed up the training +mp_start_method = "fork" diff --git a/configs/centernet/wtw_centernet_resnet34_dcnv2_150e_coco.py b/configs/centernet/wtw_centernet_resnet34_dcnv2_150e_coco.py new file mode 100644 index 00000000..e670ac06 --- /dev/null +++ b/configs/centernet/wtw_centernet_resnet34_dcnv2_150e_coco.py @@ -0,0 +1,300 @@ +TEST_NAME = "15_paper_params_resnet34_batch8" + +# DATA AND AUG +dataset_type = "CocoDataset" +data_root = "/home/aiarhipov/datasets/WTW-dataset/" + +img_norm_cfg = dict(mean=[103.53, 116.28, 123.675], std=[1.0, 1.0, 1.0], to_rgb=False) + +data = dict( + samples_per_gpu=8, + workers_per_gpu=2, + train=dict( + type=dataset_type, + ann_file="/home/aiarhipov/datasets/WTW-dataset/train/train.json", + img_prefix="train/images/", + pipeline=[ + dict(type="LoadImageFromFile", to_float32=True, color_type="color"), + dict(type="LoadAnnotations", with_bbox=True), + dict( + type="PhotoMetricDistortion", + brightness_delta=32, + contrast_range=(0.5, 1.5), + saturation_range=(0.5, 1.5), + hue_delta=18, + ), + dict( + type="RandomCenterCropPad", + crop_size=(1024, 1024), + ratios=(0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3), + mean=[0, 0, 0], + std=[1, 1, 1], + to_rgb=True, + test_pad_mode=None, + ), + dict(type="Resize", img_scale=(1024, 1024), keep_ratio=True), + dict(type="RandomFlip", flip_ratio=0.5), + dict( + type="Normalize", + mean=[103.53, 116.28, 123.675], + std=[1.0, 1.0, 1.0], + to_rgb=False, + ), + dict(type="DefaultFormatBundle"), + dict(type="Collect", keys=["img", "gt_bboxes", "gt_labels"]), + ], + data_root="/home/aiarhipov/datasets/WTW-dataset/", + classes=("box",), + ), + val_loss=dict( + type=dataset_type, + ann_file="/home/aiarhipov/datasets/WTW-dataset/test/test.json", + img_prefix="test/images/", + pipeline=[ + dict(type="LoadImageFromFile", to_float32=True, color_type="color"), + dict(type="LoadAnnotations", with_bbox=True), + dict( + type="PhotoMetricDistortion", + brightness_delta=32, + contrast_range=(0.5, 1.5), + saturation_range=(0.5, 1.5), + hue_delta=18, + ), + dict( + type="RandomCenterCropPad", + crop_size=(1024, 1024), + ratios=(0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3), + mean=[0, 0, 0], + std=[1, 1, 1], + to_rgb=True, + test_pad_mode=None, + ), + dict(type="Resize", img_scale=(1024, 1024), keep_ratio=True), + dict(type="RandomFlip", flip_ratio=0.5), + dict( + type="Normalize", + mean=[103.53, 116.28, 123.675], + std=[1.0, 1.0, 1.0], + to_rgb=False, + ), + dict(type="DefaultFormatBundle"), + dict(type="Collect", keys=["img", "gt_bboxes", "gt_labels"]), + ], + data_root="/home/aiarhipov/datasets/WTW-dataset/", + classes=("box",), + ), + val=dict( + type=dataset_type, + ann_file="/home/aiarhipov/datasets/WTW-dataset/test/test.json", + img_prefix="test/images/", + pipeline=[ + dict(type="LoadImageFromFile", to_float32=True), + dict( + type="MultiScaleFlipAug", + scale_factor=1.0, + flip=False, + transforms=[ + dict(type="Resize", keep_ratio=True), + dict( + type="RandomCenterCropPad", + ratios=None, + border=None, + mean=[0, 0, 0], + std=[1, 1, 1], + to_rgb=True, + test_mode=True, + test_pad_mode=["logical_or", 31], + test_pad_add_pix=1, + ), + dict(type="RandomFlip"), + dict( + type="Normalize", + mean=[103.53, 116.28, 123.675], + std=[1.0, 1.0, 1.0], + to_rgb=False, + ), + dict(type="DefaultFormatBundle"), + dict( + type="Collect", + meta_keys=( + "filename", + "ori_filename", + "ori_shape", + "img_shape", + "pad_shape", + "scale_factor", + "flip", + "flip_direction", + "img_norm_cfg", + "border", + ), + keys=["img"], + ), + ], + ), + ], + data_root="/home/aiarhipov/datasets/WTW-dataset/", + classes=("box",), + ), + test=dict( + type=dataset_type, + ann_file="/home/aiarhipov/datasets/WTW-dataset/test/test.json", + img_prefix="test/images/", + pipeline=[ + dict(type="LoadImageFromFile", to_float32=True), + dict( + type="MultiScaleFlipAug", + scale_factor=1.0, + flip=False, + transforms=[ + dict(type="Resize", keep_ratio=True), + dict( + type="RandomCenterCropPad", + ratios=None, + border=None, + mean=[0, 0, 0], + std=[1, 1, 1], + to_rgb=True, + test_mode=True, + test_pad_mode=["logical_or", 31], + test_pad_add_pix=1, + ), + dict(type="RandomFlip"), + dict( + type="Normalize", + mean=[103.53, 116.28, 123.675], + std=[1.0, 1.0, 1.0], + to_rgb=False, + ), + dict(type="DefaultFormatBundle"), + dict( + type="Collect", + meta_keys=( + "filename", + "ori_filename", + "ori_shape", + "img_shape", + "pad_shape", + "scale_factor", + "flip", + "flip_direction", + "img_norm_cfg", + "border", + ), + keys=["img"], + ), + ], + ), + ], + data_root="/home/aiarhipov/datasets/WTW-dataset/", + classes=("box",), + ), +) + + +# MODEL CenterNet(dcnv2) without changes +load_from = None +resume_from = None + +model = dict( + type="CenterNet", + backbone=dict( + type="ResNet", + depth=34, + norm_eval=False, + norm_cfg=dict(type="BN"), + init_cfg=dict(type="Pretrained", checkpoint="torchvision://resnet34"), + ), + neck=dict( + type="CTResNetNeck", + in_channel=512, + num_deconv_filters=(256, 128, 64), + num_deconv_kernels=(4, 4, 4), + use_dcn=True, + ), + bbox_head=dict( + type="CenterNetHead", + num_classes=1, + in_channel=64, + feat_channel=64, + loss_center_heatmap=dict(type="GaussianFocalLoss", loss_weight=1.0), + loss_wh=dict(type="L1Loss", loss_weight=0.1), + loss_offset=dict(type="L1Loss", loss_weight=1.0), + ), + train_cfg=None, + test_cfg=dict(topk=3000, local_maximum_kernel=1, max_per_img=3000), +) + + +# GPU +gpu_ids = [6] +device = "cuda" + + +# OPTIMIZATION +optimizer = dict(type="SGD", lr=0.00125, momentum=0.9, weight_decay=0.0001) +# Based on the default settings of modern detectors, the SGD effect is better +# than the Adam in the source code, so we use SGD default settings and +# if you use adam+lr5e-4, the map is 29.1. +optimizer_config = dict(grad_clip=dict(max_norm=35, norm_type=2)) + + +# LEARNING POLICY +runner = dict(type="EpochBasedRunner", max_epochs=150) # the real epoch is 28*5=140 + +# Based on the default settings of modern detectors, we added warmup settings. +lr_config = dict( + policy="step", + warmup="linear", + warmup_iters=1000, + warmup_ratio=0.001, + step=[90, 120], # the real step is [18*5, 24*5] +) + +# NOTE: `auto_scale_lr` is for automatically scaling LR, +# USER SHOULD NOT CHANGE ITS VALUES. +# base_batch_size = (8 GPUs) x (16 samples per GPU) +auto_scale_lr = dict(enable=False, base_batch_size=16) + + +# LOGGING +work_dir = f"/home/aiarhipov/centernet/exps/{TEST_NAME}" + +log_config = dict( + interval=1000, + hooks=[ + dict(type="TextLoggerHook"), + dict(type="TensorboardLoggerHook"), + dict( + type="MMDetWandbHook", + init_kwargs={ + "project": "CenterNet", + "entity": "centernet", + "name": TEST_NAME, + }, + interval=1000, + log_checkpoint=True, + log_checkpoint_metadata=True, + num_eval_images=15, + ), + ], +) +log_level = "INFO" + + +# EVALUATION +evaluation = dict(interval=30, metric="bbox") +checkpoint_config = dict(interval=30) + + +# RUNTIME +seed = 0 + +custom_hooks = [dict(type="NumClassCheckHook")] +dist_params = dict(backend="nccl") +workflow = [("train", 1), ("val", 1)] + +# disable opencv multithreading to avoid system being overloaded +opencv_num_threads = 0 +# set multi-process start method as `fork` to speed up the training +mp_start_method = "fork" diff --git a/configs/centernet/wtw_quad_cycle_centernet_dla34_dcnv2_150e_coco.py b/configs/centernet/wtw_quad_cycle_centernet_dla34_dcnv2_150e_coco.py new file mode 100644 index 00000000..8401b89b --- /dev/null +++ b/configs/centernet/wtw_quad_cycle_centernet_dla34_dcnv2_150e_coco.py @@ -0,0 +1,327 @@ +TEST_NAME = "32_quad_long" +EVAL_LAG = 150 +CHECKPOINT_LAG = 1 +EPOCHS = 150 +LR = 0.00125 +BACKBONE = "DLANetMMDet3D" +BATCH = 8 +local_maximum_kernel = 1 +ITER_PERIOD = 2 +TAGS = [ + f"local_maximum_kernel={local_maximum_kernel}", + "1hm", + f"{EPOCHS}_epoch", + f"{LR}_lr", + f"{EVAL_LAG}_eval_lag", + f"{CHECKPOINT_LAG}_chkpt_lag", + f"{BACKBONE}_backbone", + f"{BATCH}_batch", +] +# DATA AND AUG +dataset_type = "CocoDataset" +data_root = "/home/aiarhipov/datasets/WTW-dataset/" + +img_norm_cfg = dict(mean=[103.53, 116.28, 123.675], std=[1.0, 1.0, 1.0], to_rgb=False) + +data = dict( + # train_dataloader=dict(shuffle=), + samples_per_gpu=BATCH, + workers_per_gpu=2, + train=dict( + type=dataset_type, + ann_file="/home/aiarhipov/datasets/WTW-dataset/train/train_4x10.json", + img_prefix="train/images/", + pipeline=[ + dict(type="LoadImageFromFile", to_float32=True, color_type="color"), + dict(type="LoadAnnotations", with_bbox=True), + dict( + type="PhotoMetricDistortion", + brightness_delta=32, + contrast_range=(0.5, 1.5), + saturation_range=(0.5, 1.5), + hue_delta=18, + ), + dict( + type="RandomCenterCropPad", + crop_size=(1024, 1024), + ratios=(0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3), + mean=[0, 0, 0], + std=[1, 1, 1], + to_rgb=True, + test_pad_mode=None, + ), + dict(type="Resize", img_scale=(1024, 1024), keep_ratio=True), + dict(type="RandomFlip", flip_ratio=0.5), + dict( + type="Normalize", + mean=[103.53, 116.28, 123.675], + std=[1.0, 1.0, 1.0], + to_rgb=False, + ), + dict(type="DefaultFormatBundle"), + dict(type="Collect", keys=["img", "gt_bboxes", "gt_labels"]), + ], + data_root="/home/aiarhipov/datasets/WTW-dataset/", + classes=("box",), + ), + val_loss=dict( + type=dataset_type, + ann_file="/home/aiarhipov/datasets/WTW-dataset/test/test_4x10.json", + img_prefix="test/images/", + pipeline=[ + dict(type="LoadImageFromFile", to_float32=True, color_type="color"), + dict(type="LoadAnnotations", with_bbox=True), + dict( + type="PhotoMetricDistortion", + brightness_delta=32, + contrast_range=(0.5, 1.5), + saturation_range=(0.5, 1.5), + hue_delta=18, + ), + dict( + type="RandomCenterCropPad", + crop_size=(1024, 1024), + ratios=(0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3), + mean=[0, 0, 0], + std=[1, 1, 1], + to_rgb=True, + test_pad_mode=None, + ), + dict(type="Resize", img_scale=(1024, 1024), keep_ratio=True), + dict(type="RandomFlip", flip_ratio=0.5), + dict( + type="Normalize", + mean=[103.53, 116.28, 123.675], + std=[1.0, 1.0, 1.0], + to_rgb=False, + ), + dict(type="DefaultFormatBundle"), + dict(type="Collect", keys=["img", "gt_bboxes", "gt_labels"]), + ], + data_root="/home/aiarhipov/datasets/WTW-dataset/", + classes=("box",), + ), + val=dict( + type=dataset_type, + ann_file="/home/aiarhipov/datasets/WTW-dataset/test/test_4x10.json", + img_prefix="test/images/", + pipeline=[ + dict(type="LoadImageFromFile", to_float32=True), + dict( + type="MultiScaleFlipAug", + scale_factor=1.0, + flip=False, + transforms=[ + dict(type="Resize", keep_ratio=True), + dict( + type="RandomCenterCropPad", + ratios=None, + border=None, + mean=[0, 0, 0], + std=[1, 1, 1], + to_rgb=True, + test_mode=True, + test_pad_mode=["logical_or", 31], + test_pad_add_pix=1, + ), + dict(type="RandomFlip"), + dict( + type="Normalize", + mean=[103.53, 116.28, 123.675], + std=[1.0, 1.0, 1.0], + to_rgb=False, + ), + dict(type="DefaultFormatBundle"), + dict( + type="Collect", + meta_keys=( + "filename", + "ori_filename", + "ori_shape", + "img_shape", + "pad_shape", + "scale_factor", + "flip", + "flip_direction", + "img_norm_cfg", + "border", + ), + keys=["img"], + ), + ], + ), + ], + data_root="/home/aiarhipov/datasets/WTW-dataset/", + classes=("box",), + ), + test=dict( + type=dataset_type, + ann_file="/home/aiarhipov/datasets/WTW-dataset/test/test_4x10.json", + img_prefix="test/images/", + pipeline=[ + dict(type="LoadImageFromFile", to_float32=True), + dict( + type="MultiScaleFlipAug", + scale_factor=1.0, + flip=False, + transforms=[ + dict(type="Resize", keep_ratio=True), + dict( + type="RandomCenterCropPad", + ratios=None, + border=None, + mean=[0, 0, 0], + std=[1, 1, 1], + to_rgb=True, + test_mode=True, + test_pad_mode=["logical_or", 31], + test_pad_add_pix=1, + ), + dict(type="RandomFlip"), + dict( + type="Normalize", + mean=[103.53, 116.28, 123.675], + std=[1.0, 1.0, 1.0], + to_rgb=False, + ), + dict(type="DefaultFormatBundle"), + dict( + type="Collect", + meta_keys=( + "filename", + "ori_filename", + "ori_shape", + "img_shape", + "pad_shape", + "scale_factor", + "flip", + "flip_direction", + "img_norm_cfg", + "border", + ), + keys=["img"], + ), + ], + ), + ], + data_root="/home/aiarhipov/datasets/WTW-dataset/", + classes=("box",), + ), +) + + +# MODEL CycleCenterNet(dcnv2) DLANetMMDet3D, +load_from = None +resume_from = "/home/aiarhipov/centernet/exps/32_quad_long/epoch_85.pth" + +model = dict( + type="CenterNet", + backbone=dict( + type=BACKBONE, + depth=34, + norm_cfg=dict(type="BN"), + init_cfg=dict( + type="Pretrained", + checkpoint="http://dl.yf.io/dla/models/imagenet/dla34%2Btricks-24a49e58.pth", + ), + ), + neck=dict( + type="CTResNetNeck", + in_channel=512, + num_deconv_filters=(256, 128, 64), + num_deconv_kernels=(4, 4, 4), + use_dcn=True, + ), + bbox_head=dict( + type="CycleCenterNetHead", + in_channel=64, + feat_channel=64, + num_classes=1, + loss_center_heatmap=dict(type="GaussianFocalLoss", loss_weight=1.0), + loss_offset=dict(type="L1Loss", loss_weight=1.0), + loss_c2v=dict(type="L1Loss", loss_weight=1.0), + loss_v2c=dict(type="L1Loss", loss_weight=0.5), + ), + train_cfg=None, + test_cfg=dict( + topk=3000, + local_maximum_kernel=local_maximum_kernel, + max_per_img=3000, + ), +) + + +# GPU +gpu_ids = [5] +device = "cuda" + + +# OPTIMIZATION +optimizer = dict(type="SGD", lr=LR, momentum=0.9, weight_decay=0.0001) +# Based on the default settings of modern detectors, the SGD effect is better +# than the Adam in the source code, so we use SGD default settings and +# if you use adam+lr5e-4, the map is 29.1. +optimizer_config = dict(grad_clip=dict(max_norm=35, norm_type=2)) + + +# LEARNING POLICY +runner = dict(type="EpochBasedRunner", max_epochs=EPOCHS) # the real epoch is 28*5=140 + +# Based on the default settings of modern detectors, we added warmup settings. +lr_config = dict( + policy="step", + warmup="linear", + warmup_iters=1000, + warmup_ratio=0.001, + step=[90, 120], # the real step is [18*5, 24*5] +) + +# NOTE: `auto_scale_lr` is for automatically scaling LR, +# USER SHOULD NOT CHANGE ITS VALUES. +# base_batch_size = (8 GPUs) x (16 samples per GPU) +auto_scale_lr = dict(enable=False, base_batch_size=16) + + +# LOGGING +work_dir = f"/home/aiarhipov/centernet/exps/{TEST_NAME}" +INTERVAL = int(10976 / (ITER_PERIOD * BATCH)) + (10976 % (ITER_PERIOD * BATCH) > 0) +log_config = dict( + interval=INTERVAL, + hooks=[ + dict(type="TextLoggerHook"), + dict(type="TensorboardLoggerHook"), + dict( + type="MMDetWandbHook", + init_kwargs={ + "project": "CenterNet", + "entity": "centernet", + "name": TEST_NAME, + "dir": "/home/aiarhipov/centernet/exps/wandb", + "tags": TAGS, + }, + interval=INTERVAL, + log_checkpoint=True, + log_checkpoint_metadata=False, + num_eval_images=5, + ), + ], +) +log_level = "INFO" + + +# EVALUATION +evaluation = dict(interval=EVAL_LAG, metric="bbox") +checkpoint_config = dict(interval=CHECKPOINT_LAG, max_keep_ckpts=1) + + +# RUNTIME +seed = 0 + +custom_hooks = [dict(type="NumClassCheckHook")] +dist_params = dict(backend="nccl") +workflow = [("train", 1), ("val", 1)] + +# disable opencv multithreading to avoid system being overloaded +opencv_num_threads = 0 +# set multi-process start method as `fork` to speed up the training +mp_start_method = "fork" From 71ca9f13cf6e17f7ff95f18688e24480fbab7bbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=85=D0=B8=D0=BF=D0=BE=D0=B2=20=D0=90=D0=BB?= =?UTF-8?q?=D0=B5=D0=BA=D1=81=D0=B0=D0=BD=D0=B4=D1=80?= <32227207+ArchieAlexArkhipov@users.noreply.github.com> Date: Mon, 26 Jun 2023 19:53:48 +0300 Subject: [PATCH 22/25] add configs to README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 89ff32fc..5d4a6744 100644 --- a/README.md +++ b/README.md @@ -13,13 +13,13 @@ This paper tackles the problem of table structure parsing (TSP) from images in t ## Models -[**Link**](https://drive.google.com/file/d/1ZQZycYwWXMlfZnvC9hFUCtlqd4eOwP8v/view?usp=share_link) to download CenterNet(ResNet backbone) +[**Link**](https://drive.google.com/file/d/1ZQZycYwWXMlfZnvC9hFUCtlqd4eOwP8v/view?usp=share_link) to download model CenterNet(ResNet backbone). [**Config**](https://github.com/ArchieAlexArkhipov/Cycle-CenterNet/blob/master/configs/centernet/wtw_centernet_resnet34_dcnv2_150e_coco.py) -[**Link**](https://drive.google.com/file/d/1aZ2IF0tQq1Ino4QAsmrk8jcJ85urqsFE/view?usp=share_link) to download CenterNet(DLA backbone) +[**Link**](https://drive.google.com/file/d/1aZ2IF0tQq1Ino4QAsmrk8jcJ85urqsFE/view?usp=share_link) to download model CenterNet(DLA backbone). [**Config**](https://github.com/ArchieAlexArkhipov/Cycle-CenterNet/blob/master/configs/centernet/wtw_centernet_dla34_dcnv2_150e_coco.py) -[**Link**](https://drive.google.com/file/d/1oeqtA84eF_KJi953f8qX8cggDBjOVS3A/view?usp=share_link) to download Cycle-CenterNet(DLA backbone) trained on bounding boxes +[**Link**](https://drive.google.com/file/d/1oeqtA84eF_KJi953f8qX8cggDBjOVS3A/view?usp=share_link) to download model Cycle-CenterNet(DLA backbone) trained on bounding boxes. [**Config**](https://github.com/ArchieAlexArkhipov/Cycle-CenterNet/blob/master/configs/centernet/wtw_bbox_cycle_centernet_dla34_dcnv2_150e_coco.py) -[**Link**](https://drive.google.com/file/d/1taFK_co-9ofbL1pivhKrqHf8-icb1ZaK/view?usp=share_link) **to download Cycle-CenterNet(DLA backbone) trained on bounding quadrangles** +[**Link**](https://drive.google.com/file/d/1taFK_co-9ofbL1pivhKrqHf8-icb1ZaK/view?usp=share_link) **to download model Cycle-CenterNet(DLA backbone) trained on bounding quadrangles**. [**Config**](https://github.com/ArchieAlexArkhipov/Cycle-CenterNet/blob/master/configs/centernet/wtw_quad_cycle_centernet_dla34_dcnv2_150e_coco.py) ## Results From b517cdf3d4d3cf4d4795aa56101c63a67b92b4c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=85=D0=B8=D0=BF=D0=BE=D0=B2=20=D0=90=D0=BB?= =?UTF-8?q?=D0=B5=D0=BA=D1=81=D0=B0=D0=BD=D0=B4=D1=80?= <32227207+ArchieAlexArkhipov@users.noreply.github.com> Date: Mon, 26 Jun 2023 19:57:03 +0300 Subject: [PATCH 23/25] add installation to README.md --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index 5d4a6744..7e4ca09a 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,20 @@ This paper tackles the problem of table structure parsing (TSP) from images in t ## Post-processing +## Installation + +```conda create --name openmmlab python=3.8 -y``` + +```conda activate openmmlab``` + +```pip install torch==1.8.1+cu101 torchvision==0.9.1+cu101 torchaudio==0.8.1 -f https://download.pytorch.org/whl/torch_stable.html``` + +```pip install mmcv-full==1.6.2 -f https://download.openmmlab.com/mmcv/dist/cu101/torch1.8/index.html``` + +```pip install -e .``` + +```conda install -n openmmlab ipykernel --update-deps --force-reinstall``` + # MMDetection README
From 537b507cb13906d166d3cfd1c3422490bf7fbd9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=85=D0=B8=D0=BF=D0=BE=D0=B2=20=D0=90=D0=BB?= =?UTF-8?q?=D0=B5=D0=BA=D1=81=D0=B0=D0=BD=D0=B4=D1=80?= <32227207+ArchieAlexArkhipov@users.noreply.github.com> Date: Mon, 26 Jun 2023 19:58:47 +0300 Subject: [PATCH 24/25] Update README.md --- README.md | 7 ------- 1 file changed, 7 deletions(-) diff --git a/README.md b/README.md index 7e4ca09a..e9c6a7ad 100644 --- a/README.md +++ b/README.md @@ -21,13 +21,6 @@ This paper tackles the problem of table structure parsing (TSP) from images in t [**Link**](https://drive.google.com/file/d/1taFK_co-9ofbL1pivhKrqHf8-icb1ZaK/view?usp=share_link) **to download model Cycle-CenterNet(DLA backbone) trained on bounding quadrangles**. [**Config**](https://github.com/ArchieAlexArkhipov/Cycle-CenterNet/blob/master/configs/centernet/wtw_quad_cycle_centernet_dla34_dcnv2_150e_coco.py) - -## Results - -## Preprocessing - -## Post-processing - ## Installation ```conda create --name openmmlab python=3.8 -y``` From 72e54bc3c5d82215eff6bcea26108ba496bd1dc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=85=D0=B8=D0=BF=D0=BE=D0=B2=20=D0=90=D0=BB?= =?UTF-8?q?=D0=B5=D0=BA=D1=81=D0=B0=D0=BD=D0=B4=D1=80?= <32227207+ArchieAlexArkhipov@users.noreply.github.com> Date: Wed, 28 Jun 2023 10:43:01 +0300 Subject: [PATCH 25/25] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e9c6a7ad..85fe7b23 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ The unofficial Cycle-CenterNet architecture repository, based on the MMDetection > [**Parsing table structures in the wild**]([http://openaccess.thecvf.com/content/ICCV2021/papers/Long_Parsing_Table_Structures_in_the_Wild_ICCV_2021_paper.pdf]), > Long, R., Wang, W., Xue, N., Gao, F., Yang, Z., Wang, Y., & Xia, G. S. -Contact: [arkhipov.ai@phystech.edu](mailto:arkhipov.ai@phystech.edu). Any questions or discussions are welcomed! +Contact: [arkhipov.ai@phystech.edu](mailto:arkhipov.ai@phystech.edu). Any questions or discussions are welcome! ## Abstract This paper tackles the problem of table structure parsing (TSP) from images in the wild. In contrast to existing studies that mainly focus on parsing well-aligned tabular images with simple layouts from scanned PDF documents, we aim to establish a practical table structure parsing system for real-world scenarios where tabular input images are taken or scanned with severe deformation, bending or occlusions. For designing such a system, we propose an approach named Cycle-CenterNet on the top of CenterNet with a novel cycle-pairing module to simultaneously detect and group tabular cells into structured tables. In the cycle-pairing module, a new pairing loss function is proposed for the network training. Alongside with our Cycle-CenterNet, we also present a large-scale dataset, named Wired Table in the Wild (WTW), which includes well-annotated structure parsing of multiple style tables in several scenes like photo, scanning files, web pages, etc.. In experiments, we demonstrate that our Cycle-CenterNet consistently achieves the best accuracy of table structure parsing on the new WTW dataset by 24.6% absolute improvement evaluated by the TEDS metric. A more comprehensive experimental analysis also validates the advantages of our proposed methods for the TSP task.