SSD-2（代码部分介绍）

# Copyright 2016 Paul Balanca. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Definition of 300 VGG-based SSD network. This model was initially introduced in:
SSD: Single Shot MultiBox Detector
Wei Liu, Dragomir Anguelov, Dumitru Erhan, Christian Szegedy, Scott Reed,
Cheng-Yang Fu, Alexander C. Berg
https://arxiv.org/abs/1512.02325 Two variants of the model are defined: the 300x300 and 512x512 models, the
latter obtaining a slightly better accuracy on Pascal VOC. Usage:
with slim.arg_scope(ssd_vgg.ssd_vgg()):
outputs, end_points = ssd_vgg.ssd_vgg(inputs) This network port of the original Caffe model. The padding in TF and Caffe
is slightly different, and can lead to severe accuracy drop（精度严重下降） if not taken care
in a correct way! In Caffe, the output size of convolution and pooling layers are computing as
following: h_o = (h_i + 2 * pad_h - kernel_h) / stride_h + 1 Nevertheless（然而）, there is a subtle（微妙的） difference between both for stride > 1. In
the case of convolution（在卷积的情况下）:
top_size = floor((bottom_size + 2*pad - kernel_size) / stride) + 1
whereas for pooling:
top_size = ceil((bottom_size + 2*pad - kernel_size) / stride) + 1
Hence implicitely allowing some additional padding even if pad = 0（隐含的允许一些额外的填充）. This
behaviour explains why pooling with stride and kernel of size 2 are behaving
the same way in TensorFlow and Caffe. Nevertheless, this is not the case anymore for other kernel sizes（）对于其他kernel，情况就不同了, hence
motivating the use of special padding layer for controlling these side-effects.（鼓励使用特殊的填充层来控制这种副作用） @@ssd_vgg_300
"""
import math
from collections import namedtuple import numpy as np
import tensorflow as tf import tf_extended as tfe
from nets import custom_layers
from nets import ssd_common slim = tf.contrib.slim # =========================================================================== #
# SSD class definition.
# =========================================================================== #
#collections模块的namedtuple子类不仅可以使用item的index访问item，
# 还可以通过item的name进行访问可以将namedtuple理解为c中的struct结构，
# 其首先将各个item命名，然后对每个item赋予数据
# nametuple(tuple名字，域名)
SSDParams = namedtuple('SSDParameters', ['img_shape', #输入图像大小
'num_classes', #类+1（背景）
'no_annotation_label', #无标注标签？？？？
'feat_layers', #特征层
'feat_shapes', #特征层形状
'anchor_size_bounds', #锚点框大小上下边界，相对于原图的比例值
'anchor_sizes', #初始锚点框尺寸
'anchor_ratios', #锚点框长宽比
'anchor_steps', #feature map相对于原图的缩小倍数，后面会解释
'anchor_offset', #锚点框中心的偏移
'normalizations', #是否正则化
'prior_scaling' ##特征图上每个目标与参考框间的尺寸缩放（y,x,h,w）解码时用到
]) class SSDNet(object):
"""Implementation of the SSD VGG-based 300 network. The default features layers with 300x300 image input are:
conv4 ==> 38 x 38
conv7 ==> 19 x 19
conv8 ==> 10 x 10
conv9 ==> 5 x 5
conv10 ==> 3 x 3
conv11 ==> 1 x 1
The default image size used to train this network is 300x300.
"""
default_params = SSDParams( #默认参数
img_shape=(300, 300),
num_classes=21, #类数 + 1（背景）
no_annotation_label=21, #同上
feat_layers=['block4', 'block7', 'block8', 'block9', 'block10', 'block11'], #特征层名字
feat_shapes=[(38, 38), (19, 19), (10, 10), (5, 5), (3, 3), (1, 1)], #特征层尺寸
anchor_size_bounds=[0.15, 0.90], #第一层feature map的default box缩放比例Sk，大小为：300x0.15,300x0.9
# anchor_size_bounds=[0.20, 0.90], #论文中是300x0.2,300x0.9 #anchor的大小，一共6个比例，下面的是原图根据比例计算后的得到的实际anchor大小
#4,6，6，6，4，4（每层feature map的dafault box的个数）
#长宽都是有计算公式的，得到Sk后，通过公式得到h，w
anchor_sizes=[(21., 45.), #h，w
(45., 99.),
(99., 153.),
(153., 207.),
(207., 261.),
(261., 315.)], #越小的anchor box,得到的信息越大，这个是相对于原图的大小，越来越大
# anchor_sizes=[(30., 60.),
# (60., 111.),
# (111., 162.),
# (162., 213.),
# (213., 264.),
# (264., 315.)], ##每个特征层上的每个特征点预测的box长宽比及数量，例如：[2, .5]：（1:1）、（2:1）、（1:2）、（1:1），这里是把重复的省去了
#实际上是有4个default box的
anchor_ratios=[[2, .5], #block4: def_boxes:4
[2, .5, 3, 1./3], #def_boxes:6 （ratios中的4个+默认的1:1+额外增加的一个（S'k）=6）
[2, .5, 3, 1./3], #def_boxes:6
[2, .5, 3, 1./3], #def_boxes:6
[2, .5], #def_boxes:4
[2, .5]], #def_boxes:4
anchor_steps=[8, 16, 32, 64, 100, 300], #8x38=304,16x19=304,32x10=320,64x5=320,100x3=300,1x300=300
anchor_offset=0.5,
#是否归一化，大于0则进行，否则不做归一化；
# 目前看来只对block_4进行正则化，因为该层比较靠前，其norm(范数)较大，需做L2正则化
# （仅仅对每个像素在channel维度做归一化）以保证和后面检测层差异不是很大；
normalizations=[20, -1, -1, -1, -1, -1],
prior_scaling=[0.1, 0.1, 0.2, 0.2] #特征图上每个目标与参考框间的尺寸缩放（y,x,h,w）解码时用到
) def __init__(self, params=None): #网络参数初始化
"""
Init the SSD net with some parameters. Use the default ones if none provided.
"""
if isinstance(params, SSDParams): #是否有参数输入，是则用输入的，否则使用默认的
self.params = params #isinstance是python的內建函数，如果参数1与参数2的类型相同则返回true；
else: #
self.params = SSDNet.default_params # ======================================================================= #
#定义网络模型
def net(self, inputs,
is_training=True, #是否训练
update_feat_shapes=True, #是否更新特征层的尺寸
dropout_keep_prob=0.5, ##dropout=0.5
prediction_fn=slim.softmax, #采用softmax预测结果
reuse=None,
scope='ssd_300_vgg'): #网络名：ssd_300_vgg（基础网络时VGG，输入训练图像size是300x300）
"""
SSD network definition.
"""
#网络输入参数
r = ssd_net(inputs,
num_classes=self.params.num_classes,
feat_layers=self.params.feat_layers,
anchor_sizes=self.params.anchor_sizes,
anchor_ratios=self.params.anchor_ratios,
normalizations=self.params.normalizations,
is_training=is_training,
dropout_keep_prob=dropout_keep_prob,
prediction_fn=prediction_fn,
reuse=reuse,
scope=scope)
# Update feature shapes (try at least!)
# 下面这步我的理解就是让读者自行更改特征层的输入，未必论文中介绍的那几个block
if update_feat_shapes: #是否更新特征层图像尺寸？
#输入特征层图像尺寸以及inputs（应该是预测的特征尺寸），输出更新后的特征图尺寸列表
shapes = ssd_feat_shapes_from_net(r[0], self.params.feat_shapes)
#将更新的特征图尺寸shapes替换当前的特征图尺寸
self.params = self.params._replace(feat_shapes=shapes)
return r ##更新网络输入参数r # 定义权重衰减=0.0005，L2正则化项系数；数据类型是NHWC:[batch, height, width, channels]
def arg_scope(self, weight_decay=0.0005, data_format='NHWC'):
"""Network arg_scope.
"""
return ssd_arg_scope(weight_decay, data_format=data_format) def arg_scope_caffe(self, caffe_scope):
"""Caffe arg_scope used for weights importing.
"""
return ssd_arg_scope_caffe(caffe_scope) # ======================================================================= #
##更新特征形状尺寸（来自预测结果）
def update_feature_shapes(self, predictions):
"""Update feature shapes from predictions collection (Tensor or Numpy
array).
"""
shapes = ssd_feat_shapes_from_net(predictions, self.params.feat_shapes)
self.params = self.params._replace(feat_shapes=shapes)
#输入原始图像尺寸；返回每个特征层每个参考锚点框的位置及尺寸信息（x,y,h,w）
def anchors(self, img_shape, dtype=np.float32):
"""Compute the default anchor boxes, given an image shape.
"""
return ssd_anchors_all_layers(img_shape,
self.params.feat_shapes,
self.params.anchor_sizes,
self.params.anchor_ratios,
self.params.anchor_steps,
self.params.anchor_offset,
dtype)
#编码，用于将标签信息，真实目标信息和锚点框信息编码在一起；得到预测真实框到参考框的转换值
def bboxes_encode(self, labels, bboxes, anchors,
scope=None):
"""Encode labels and bounding boxes.
"""
return ssd_common.tf_ssd_bboxes_encode(
labels, bboxes, anchors,
self.params.num_classes,
self.params.no_annotation_label, #未标注的标签（应该代表背景）
ignore_threshold=0.5, #IOU筛选阈值
prior_scaling=self.params.prior_scaling, #特征图目标与参考框间的尺寸缩放（0.1,0.1,0.2,0.2）
scope=scope)
#解码，用锚点框信息，锚点框与预测真实框间的转换值，得到真实的预测框（ymin,xmin,ymax,xmax）
def bboxes_decode(self, feat_localizations, anchors,
scope='ssd_bboxes_decode'):
"""Encode labels and bounding boxes.
"""
return ssd_common.tf_ssd_bboxes_decode(
feat_localizations, anchors,
prior_scaling=self.params.prior_scaling,
scope=scope)
#通过SSD网络，得到检测到的bbox
def detected_bboxes(self, predictions, localisations,
select_threshold=None, nms_threshold=0.5,
clipping_bbox=None, top_k=400, keep_top_k=200):
"""Get the detected bounding boxes from the SSD network output.
"""
# Select top_k bboxes from predictions, and clip
# 选取top_k=400个框，并对框做修建（超出原图尺寸范围的切掉） # 得到对应某个类别的得分值以及bbox
rscores, rbboxes = \
ssd_common.tf_ssd_bboxes_select(predictions, localisations,
select_threshold=select_threshold,
num_classes=self.params.num_classes)
#按照得分高低，筛选出400个bbox和对应得分
rscores, rbboxes = \
tfe.bboxes_sort(rscores, rbboxes, top_k=top_k)
# Apply NMS algorithm.
# 应用非极大值抑制，去掉与得分最高的bbox的重叠率大于nms_threshold=0.5的，保留200个
rscores, rbboxes = \
tfe.bboxes_nms_batch(rscores, rbboxes,
nms_threshold=nms_threshold,
keep_top_k=keep_top_k)
if clipping_bbox is not None:
rbboxes = tfe.bboxes_clip(clipping_bbox, rbboxes)
return rscores, rbboxes #返回裁剪好的bbox和对应得分 # 尽管一个ground truth可以与多个先验框匹配，但是ground truth相对先验框还是太少了，
# 所以负样本相对正样本会很多。为了保证正负样本尽量平衡，SSD采用了hard negative mining，
# 就是对负样本进行抽样，抽样时按照置信度误差（预测背景的置信度越小（预测背景，但实际上不是背景的概率很大），误差越大）进行降序排列，
# 选取误差的较大的top-k作为训练的负样本，以保证正负样本比例接近1:3
def losses(self, logits, localisations,
gclasses, glocalisations, gscores,
match_threshold=0.5,
negative_ratio=3.,
alpha=1.,
label_smoothing=0.,
scope='ssd_losses'):
"""
Define the SSD network losses.
"""
return ssd_losses(logits, localisations,
gclasses, glocalisations, gscores,
match_threshold=match_threshold,
negative_ratio=negative_ratio,
alpha=alpha,
label_smoothing=label_smoothing,
scope=scope) # =========================================================================== #
# SSD tools...
# =========================================================================== #
# ？？？？
def ssd_size_bounds_to_values(size_bounds,
n_feat_layers,
img_shape=(300, 300)):
"""
Compute the reference sizes of the anchor boxes from relative bounds.
The absolute values are measured in pixels, based on the network
default size (300 pixels). This function follows the computation performed in the original
implementation of SSD in Caffe. Return:
list of list containing the absolute sizes at each scale. For each scale,
the ratios only apply to the first value.
"""
assert img_shape[0] == img_shape[1] img_size = img_shape[0]
min_ratio = int(size_bounds[0] * 100)
max_ratio = int(size_bounds[1] * 100)
step = int(math.floor((max_ratio - min_ratio) / (n_feat_layers - 2)))
# Start with the following smallest sizes.
sizes = [[img_size * size_bounds[0] / 2, img_size * size_bounds[0]]]
for ratio in range(min_ratio, max_ratio + 1, step):
sizes.append((img_size * ratio / 100.,
img_size * (ratio + step) / 100.))
return sizes # 得到更新后的特征尺寸list
def ssd_feat_shapes_from_net(predictions, default_shapes=None):
"""Try to obtain the feature shapes from the prediction layers. The latter
can be either a Tensor or Numpy ndarray. Return:
如果预测没有完全成型，就是用默认值
list of feature shapes. Default values if predictions shape not fully
determined.
"""
feat_shapes = []
for l in predictions: #l：预测的特征形状
# Get the shape, from either a np array or a tensor.
# 如果l是np.ndarray类型，则将l的形状赋给shape；否则将shape作为list
if isinstance(l, np.ndarray):
shape = l.shape
else:
shape = l.get_shape().as_list()
shape = shape[1:4]
# Problem: undetermined shape...
# 如果预测的特征尺寸未定，则使用默认的形状；否则将shape中的值赋给特征形状列表中
if None in shape:
return default_shapes
else:
feat_shapes.append(shape)
return feat_shapes #返回更新后的特征尺寸list #default box 的生成
#生成一层anchor box
def ssd_anchor_one_layer(img_shape, #原始图像shape
feat_shape, #特征图shape
sizes, #默认box大小，两个正方形，两个长方形，仅仅就是长宽比例相反，所以就两个
ratios, #默认box长宽比，list，就是那些比率列表,元素值是比例，列表长度是框的个数
step, #特征图上一步对应在原图上的跨度
offset=0.5,
dtype=np.float32):
"""Computer SSD default anchor boxes for one feature layer. Determine the relative position grid of the centers, and the relative
width and height.确定中心的相对位置网格和相对位置网格宽度和高度。 Arguments:
feat_shape: Feature shape, used for computing relative position grids;
size: Absolute reference sizes;
ratios: Ratios to use on these features;
img_shape: Image shape, used for computing height, width relatively to the
former;
offset: Grid offset. Return:
y, x, h, w: Relative x and y grids, and height and width.
"""
# Compute the position grid: simple way.
# y, x = np.mgrid[0:feat_shape[0], 0:feat_shape[1]]
# y = (y.astype(dtype) + offset) / feat_shape[0]
# x = (x.astype(dtype) + offset) / feat_shape[1]
# Weird SSD-Caffe computation using steps values...
# 归一化到原图的锚点中心坐标（x,y）;其坐标值域为(0,1)
# 计算default box中心坐标（相对于原图）
y, x = np.mgrid[0:feat_shape[0], 0:feat_shape[1]] # 对于第一个特征图（block4：38x38）；
# y=[[0,0,……0],[1,1,……1]，……[37,37,……，37]]；
# 而x=[[0,1,2……，37]，[0,1,2……，37],……[0,1,2……，37]]
y = (y.astype(dtype) + offset) * step / img_shape[0]# 将38个cell对应锚点框的y坐标偏移至每个cell中心，然后乘以相对原图缩放的比例，再除以原图
x = (x.astype(dtype) + offset) * step / img_shape[1]#可以得到在原图上，相对原图比例大小的每个锚点中心坐标x,y # Expand dims to support easy broadcasting.#将锚点中心坐标扩大维度
y = np.expand_dims(y, axis=-1) #对于第一个特征图，y的shape=38x38x1；x的shape=38x38x1
x = np.expand_dims(x, axis=-1) # Compute relative height and width.
# Tries to follow the original implementation of SSD for the order.
# 默认框的个数，该特征图上每个cell对应的锚点框数量;如：对于第一个特征图每个点预测4个锚点框（block4：38x38），2+2=4
num_anchors = len(sizes) + len(ratios)
h = np.zeros((num_anchors, ), dtype=dtype) #第一个锚点框的高h[0]=起始锚点的高/原图大小的高；例如：h[0]=21/300
w = np.zeros((num_anchors, ), dtype=dtype) #第一个锚点框的宽w[0]=起始锚点的宽/原图大小的宽；例如：w[0]=21/300
# Add first anchor boxes with ratio=1.
h[0] = sizes[0] / img_shape[0]# 添加长宽比为1的默认框
w[0] = sizes[0] / img_shape[1]
di = 1 #锚点框个数偏移
if len(sizes) > 1:
# 添加一组特殊的默认框，就是用S'k计算出来的box，长宽比为1，大小为sqrt（s（i） + s（i+1））
#第二个锚点框的高h[1]=sqrt（起始锚点的高*起始锚点的宽）/原图大小的高；例如：h[1]=sqrt(21*45)/300
h[1] = math.sqrt(sizes[0] * sizes[1]) / img_shape[0]
#第二个锚点框的高w[1]=sqrt（起始锚点的高*起始锚点的宽）/原图大小的宽；例如：w[1]=sqrt(21*45)/300
w[1] = math.sqrt(sizes[0] * sizes[1]) / img_shape[1]
di += 1
# 添加不同比例的默认框（ratios中不含1）
# #遍历长宽比例，第一个特征图，r只有两个，2和0.5；共四个锚点框size（h[0]~h[3]）
for i, r in enumerate(ratios):
# 例如：对于第一个特征图，h[0+2]=h[2]=21/300/sqrt(2);w[0+2]=w[2]=45/300*sqrt(2)
h[i+di] = sizes[0] / img_shape[0] / math.sqrt(r)
# 例如：对于第一个特征图，h[1+2]=h[3]=21/300/sqrt(0.5);w[1+2]=w[3]=45/300*sqrt(0.5)
w[i+di] = sizes[0] / img_shape[1] * math.sqrt(r)
return y, x, h, w #返回没有归一化前的锚点坐标和尺寸 #检测所有特征图中锚点框的四个坐标信息
def ssd_anchors_all_layers(img_shape, #输入原始图大小
layers_shape,#每个特征层形状尺寸
anchor_sizes,#起始特征图中框的长宽size
anchor_ratios,#锚点框长宽比列表
anchor_steps,#锚点框相对原图缩放比例
offset=0.5,#锚点中心在每个特征图cell中的偏移
dtype=np.float32):
"""Compute anchor boxes for all feature layers.
"""
layers_anchors = [] #用于存放所有特征图中锚点框位置尺寸信息
for i, s in enumerate(layers_shape):#6个特征图尺寸；如：第0个是38x38
# 分别计算每个特征图中锚点框的位置尺寸信息；
anchor_bboxes = ssd_anchor_one_layer(img_shape, s,
anchor_sizes[i],#输入：第i个特征图中起始锚点框大小；如第0个是(21., 45.)
anchor_ratios[i],#输入：第i个特征图中锚点框长宽比列表；如第0个是[2, .5]
anchor_steps[i],#输入：第i个特征图中锚点框相对原始图的缩放比；如第0个是8
offset=offset, dtype=dtype)#输入：第i个特征图中锚点框相对原始图的缩放比；如第0个是8
# 将6个特征图中每个特征图上的点对应的锚点框（6个或4个）保存
layers_anchors.append(anchor_bboxes)
return layers_anchors #返回所有特征图的锚点框尺寸信息 # =========================================================================== #
# Functional definition of VGG-based SSD 300.功能定义
# =========================================================================== #
#得到一个tensor的dim，list
def tensor_shape(x, rank=3):
"""Returns the dimensions of a tensor.
Args:
image: A N-D Tensor of shape.
Returns:
A list of dimensions. Dimensions that are statically known are python
integers,otherwise they are integer scalar tensors.
"""
if x.get_shape().is_fully_defined():
return x.get_shape().as_list()
else:
static_shape = x.get_shape().with_rank(rank).as_list()
dynamic_shape = tf.unstack(tf.shape(x), rank)
return [s if s is not None else d
for s, d in zip(static_shape, dynamic_shape)] #对指定feature layers的位置预测以及类别预测
#首先计算anchors的数量，对于位置信息，输出16通道的feature map，将其reshape为[N,W,H,num_anchors,4]。
#对于类别信息，输出84通道的feature maps，再将其reshape为[N,W,H,num_anchors,num_classes]。返回计算得到的位置和类别预测。
#返回计算得到的位置和类别预测。
def ssd_multibox_layer(inputs,#输入特征层
num_classes,#类别数
sizes,#参考先验框的尺度
ratios=[1],#默认的先验框长宽比为1
normalization=-1,#默认不做正则化
bn_normalization=False):
"""
Construct a multibox layer, return a class and localization predictions.
"""
net = inputs
if normalization > 0:#如果输入整数，则进行L2正则化
net = custom_layers.l2_normalization(net, scaling=True)#对通道所在维度进行正则化，随后乘以gamma缩放系数
# Number of anchors.
num_anchors = len(sizes) + len(ratios)#每层特征图参考先验框的个数[4,6,6,6,4,4] # Location.#每个先验框对应4个坐标信息
# 最后整个特征图所有锚点框预测目标位置 tensor为[h*w*每个cell先验框数，4]
num_loc_pred = num_anchors * 4#特征图上每个单元预测的坐标所需维度=锚点框数*4
# 通过对特征图进行3x3卷积得到位置信息和类别权重信息
loc_pred = slim.conv2d(net, num_loc_pred, [3, 3], activation_fn=None,
scope='conv_loc') #该部分是定位信息，输出维度为[特征图h,特征图w,每个单元所有锚点框坐标]
loc_pred = custom_layers.channel_to_last(loc_pred)
loc_pred = tf.reshape(loc_pred,tensor_shape(loc_pred, 4)[:-1]+[num_anchors, 4])
# Class prediction.
#特征图上每个单元预测的类别所需维度=锚点框数*种类数
num_cls_pred = num_anchors * num_classes
# 该部分是类别信息，输出维度为[特征图h,特征图w,每个单元所有锚点框对应类别信息]
##最后整个特征图所有锚点框预测类别 tensor为[h*w*每个cell先验框数，种类数]
cls_pred = slim.conv2d(net, num_cls_pred, [3, 3], activation_fn=None,scope='conv_cls')
cls_pred = custom_layers.channel_to_last(cls_pred)
cls_pred = tf.reshape(cls_pred,tensor_shape(cls_pred, 4)[:-1]+[num_anchors, num_classes])
return cls_pred, loc_pred #返回预测得到的类别和box位置 tensor #定义ssd网络结构
def ssd_net(inputs,
num_classes=SSDNet.default_params.num_classes, #分类数
feat_layers=SSDNet.default_params.feat_layers, #特征层
anchor_sizes=SSDNet.default_params.anchor_sizes,
anchor_ratios=SSDNet.default_params.anchor_ratios,
normalizations=SSDNet.default_params.normalizations,#正则化
is_training=True,
dropout_keep_prob=0.5,
prediction_fn=slim.softmax,
reuse=None,
scope='ssd_300_vgg'):
"""SSD net definition.
"""
# if data_format == 'NCHW':
# inputs = tf.transpose(inputs, perm=(0, 3, 1, 2)) # End_points collect relevant activations for external use.
end_points = {} #用于收集每一层输出结果
with tf.variable_scope(scope, 'ssd_300_vgg', [inputs], reuse=reuse):
# Original VGG-16 blocks. #VGG16网络的第一个conv，重复2次卷积，核为3x3,64个特征
net = slim.repeat(inputs, 2, slim.conv2d, 64, [3, 3], scope='conv1')
end_points['block1'] = net #conv1_2结果存入end_points，name='block1'
net = slim.max_pool2d(net, [2, 2], scope='pool1')
# Block 2. #重复2次卷积，核为3x3,128个特征
net = slim.repeat(net, 2, slim.conv2d, 128, [3, 3], scope='conv2')
end_points['block2'] = net #conv2_2结果存入end_points，name='block2'
net = slim.max_pool2d(net, [2, 2], scope='pool2')
# Block 3.#重复3次卷积，核为3x3,256个特征
net = slim.repeat(net, 3, slim.conv2d, 256, [3, 3], scope='conv3')
end_points['block3'] = net#conv3_3结果存入end_points，name='block3'
net = slim.max_pool2d(net, [2, 2], scope='pool3')
# Block 4.#重复3次卷积，核为3x3,512个特征
net = slim.repeat(net, 3, slim.conv2d, 512, [3, 3], scope='conv4')
end_points['block4'] = net #conv4_3结果存入end_points，name='block4'
net = slim.max_pool2d(net, [2, 2], scope='pool4')
# Block 5.#重复3次卷积，核为3x3,512个特征
net = slim.repeat(net, 3, slim.conv2d, 512, [3, 3], scope='conv5')
end_points['block5'] = net #conv5_3结果存入end_points，name='block5'
net = slim.max_pool2d(net, [3, 3], stride=1, scope='pool5') # Additional SSD blocks. #去掉了VGG的全连接层
# Block 6: let's dilate the hell out of it!
# 将VGG基础网络最后的池化层结果做扩展卷积（带孔卷积）；
net = slim.conv2d(net, 1024, [3, 3], rate=6, scope='conv6')
end_points['block6'] = net #conv6结果存入end_points，name='block6'
net = tf.layers.dropout(net, rate=dropout_keep_prob, training=is_training)#dropout层
# Block 7: 1x1 conv. Because the fuck.
# 将dropout后的网络做1x1卷积，输出1024特征，name='block7'
net = slim.conv2d(net, 1024, [1, 1], scope='conv7')
end_points['block7'] = net
net = tf.layers.dropout(net, rate=dropout_keep_prob, training=is_training)#将卷积后的网络继续做dropout # Block 8/9/10/11: 1x1 and 3x3 convolutions stride 2 (except lasts).
end_point = 'block8' #对上述dropout的网络做1x1卷积，然后做3x3卷积，,输出512特征图，name=‘block8’
with tf.variable_scope(end_point):
net = slim.conv2d(net, 256, [1, 1], scope='conv1x1')
net = custom_layers.pad2d(net, pad=(1, 1))
net = slim.conv2d(net, 512, [3, 3], stride=2, scope='conv3x3', padding='VALID')
end_points[end_point] = net
end_point = 'block9' #对上述网络做1x1卷积，然后做3x3卷积，输出256特征图，name=‘block9’
with tf.variable_scope(end_point):
net = slim.conv2d(net, 128, [1, 1], scope='conv1x1')
net = custom_layers.pad2d(net, pad=(1, 1))
net = slim.conv2d(net, 256, [3, 3], stride=2, scope='conv3x3', padding='VALID')
end_points[end_point] = net
end_point = 'block10' #对上述网络做1x1卷积，然后做3x3卷积，输出256特征图，name=‘block10’
with tf.variable_scope(end_point):
net = slim.conv2d(net, 128, [1, 1], scope='conv1x1')
net = slim.conv2d(net, 256, [3, 3], scope='conv3x3', padding='VALID')
end_points[end_point] = net
end_point = 'block11' #对上述网络做1x1卷积，然后做3x3卷积，输出256特征图，name=‘block11’
with tf.variable_scope(end_point):
net = slim.conv2d(net, 128, [1, 1], scope='conv1x1')
net = slim.conv2d(net, 256, [3, 3], scope='conv3x3', padding='VALID')
end_points[end_point] = net # Prediction and localisations layers.
# 预测和定位
predictions = []
logits = []
localisations = []
for i, layer in enumerate(feat_layers): #遍历特征层
with tf.variable_scope(layer + '_box'): #起个命名范围
# 做多尺度大小box预测的特征层，返回每个cell中每个先验框预测的类别p和预测的位置l
p, l = ssd_multibox_layer(end_points[layer],
num_classes,#种类数
anchor_sizes[i],#先验框尺度（同一特征图上的先验框尺度和长宽比一致）
anchor_ratios[i],#先验框长宽比
normalizations[i])#每个特征正则化信息，目前是只对第一个特征图做归一化操作；
# 把每一层的预测收集
predictions.append(prediction_fn(p))#prediction_fn为softmax，预测类别
logits.append(p)#把每个cell每个先验框预测的类别的概率值存在logits中
localisations.append(l)#预测位置信息
# 返回类别预测结果，位置预测结果，所属某个类别的概率值，以及特征层
return predictions, localisations, logits, end_points
ssd_net.default_image_size = 300 # 权重衰减系数=0.0005；其是L2正则化项的系数
def ssd_arg_scope(weight_decay=0.0005, data_format='NHWC'):
"""
Defines the VGG arg scope.
Args:
weight_decay: The l2 regularization coefficient.
Returns:
An arg_scope.
"""
with slim.arg_scope([slim.conv2d, slim.fully_connected],
activation_fn=tf.nn.relu,
weights_regularizer=slim.l2_regularizer(weight_decay),
weights_initializer=tf.contrib.layers.xavier_initializer(),
biases_initializer=tf.zeros_initializer()):
with slim.arg_scope([slim.conv2d, slim.max_pool2d],
padding='SAME',
data_format=data_format):
with slim.arg_scope([custom_layers.pad2d,
custom_layers.l2_normalization,
custom_layers.channel_to_last],
data_format=data_format) as sc:
return sc # =========================================================================== #
# Caffe scope: importing weights at initialization.
# =========================================================================== # def ssd_arg_scope_caffe(caffe_scope):
"""Caffe scope definition. Args:
caffe_scope: Caffe scope object with loaded weights. Returns:
An arg_scope.
"""
# Default network arg scope.
with slim.arg_scope([slim.conv2d],
activation_fn=tf.nn.relu,
weights_initializer=caffe_scope.conv_weights_init(),
biases_initializer=caffe_scope.conv_biases_init()):
with slim.arg_scope([slim.fully_connected],
activation_fn=tf.nn.relu):
with slim.arg_scope([custom_layers.l2_normalization],
scale_initializer=caffe_scope.l2_norm_scale_init()):
with slim.arg_scope([slim.conv2d, slim.max_pool2d],
padding='SAME') as sc:
return sc # =========================================================================== #
# SSD loss function.
# =========================================================================== #
def ssd_losses(logits, localisations, #损失函数定义为位置误差和置信度误差的加权和；
gclasses, glocalisations, gscores,
match_threshold=0.5,
negative_ratio=3.,
alpha=1., #位置误差权重系数
label_smoothing=0.,
device='/cpu:0',
scope=None):
with tf.name_scope(scope, 'ssd_losses'):
lshape = tfe.get_shape(logits[0], 5)
num_classes = lshape[-1]
batch_size = lshape[0] # Flatten out all vectors!
flogits = []
fgclasses = []
fgscores = []
flocalisations = []
fglocalisations = []
for i in range(len(logits)):
flogits.append(tf.reshape(logits[i], [-1, num_classes])) #将类别的概率值reshape成（-1,21）
fgclasses.append(tf.reshape(gclasses[i], [-1])) #真实类别
fgscores.append(tf.reshape(gscores[i], [-1])) #预测真实目标的得分
flocalisations.append(tf.reshape(localisations[i], [-1, 4])) #预测真实目标边框坐标（编码形式的值）
fglocalisations.append(tf.reshape(glocalisations[i], [-1, 4])) #用于将真实目标gt的坐标进行编码存储
# And concat the crap!
logits = tf.concat(flogits, axis=0)
gclasses = tf.concat(fgclasses, axis=0)
gscores = tf.concat(fgscores, axis=0)
localisations = tf.concat(flocalisations, axis=0)
glocalisations = tf.concat(fglocalisations, axis=0)
dtype = logits.dtype # Compute positive matching mask...
pmask = gscores > match_threshold #预测框与真实框IOU>0.5则将这个先验作为正样本
fpmask = tf.cast(pmask, dtype)
n_positives = tf.reduce_sum(fpmask) #求正样本数量N # Hard negative mining...
#为了保证正负样本尽量平衡，SSD采用了hard negative mining，
# 就是对负样本进行抽样，抽样时按照置信度误差（预测背景的置信度越小，误差越大）进行降序排列，
# 选取误差的较大的top - k作为训练的负样本，以保证正负样本比例接近1: 3
no_classes = tf.cast(pmask, tf.int32)
predictions = slim.softmax(logits) #类别预测
nmask = tf.logical_and(tf.logical_not(pmask),
gscores > -0.5)
fnmask = tf.cast(nmask, dtype)
nvalues = tf.where(nmask,
predictions[:, 0],
1. - fnmask)
nvalues_flat = tf.reshape(nvalues, [-1])
# Number of negative entries to select.
max_neg_entries = tf.cast(tf.reduce_sum(fnmask), tf.int32)
n_neg = tf.cast(negative_ratio * n_positives, tf.int32) + batch_size #负样本数量，保证是正样本3倍
n_neg = tf.minimum(n_neg, max_neg_entries)
# 抽样时按照置信度误差（预测背景的置信度越小，误差越大）进行降序排列，选取误差的较大的top-k作为训练的负样本
val, idxes = tf.nn.top_k(-nvalues_flat, k=n_neg)
max_hard_pred = -val[-1]
# Final negative mask.
nmask = tf.logical_and(nmask, nvalues < max_hard_pred)
fnmask = tf.cast(nmask, dtype) # Add cross-entropy loss.#交叉熵
with tf.name_scope('cross_entropy_pos'):
# 类别置信度误差
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits,labels=gclasses)
# 将置信度误差除以正样本数后除以batch-size
loss = tf.div(tf.reduce_sum(loss * fpmask), batch_size, name='value')
tf.losses.add_loss(loss) with tf.name_scope('cross_entropy_neg'):
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits,
labels=no_classes)
loss = tf.div(tf.reduce_sum(loss * fnmask), batch_size, name='value')
tf.losses.add_loss(loss) # Add localization loss: smooth L1, L2, ...
with tf.name_scope('localization'):
# Weights Tensor: positive mask + random negative.
weights = tf.expand_dims(alpha * fpmask, axis=-1)
# 先验框对应边界的位置预测值-真实位置；然后做Smooth L1 loss
loss = custom_layers.abs_smooth(localisations - glocalisations)
# 将上面的loss*权重（=alpha/正样本数）求和后除以batch-size
loss = tf.div(tf.reduce_sum(loss * weights), batch_size, name='value')
tf.losses.add_loss(loss)#获得置信度误差和位置误差的加权和 def ssd_losses_old(logits, localisations,
gclasses, glocalisations, gscores,
match_threshold=0.5,
negative_ratio=3.,
alpha=1.,
label_smoothing=0.,
device='/cpu:0',
scope=None):
"""Loss functions for training the SSD 300 VGG network. This function defines the different loss components of the SSD, and
adds them to the TF loss collection. Arguments:
logits: (list of) predictions logits Tensors;
localisations: (list of) localisations Tensors;
gclasses: (list of) groundtruth labels Tensors;
glocalisations: (list of) groundtruth localisations Tensors;
gscores: (list of) groundtruth score Tensors;
"""
with tf.device(device):
with tf.name_scope(scope, 'ssd_losses'):
l_cross_pos = []
l_cross_neg = []
l_loc = []
for i in range(len(logits)):
dtype = logits[i].dtype
with tf.name_scope('block_%i' % i):
# Sizing weight...
wsize = tfe.get_shape(logits[i], rank=5)
wsize = wsize[1] * wsize[2] * wsize[3] # Positive mask.
pmask = gscores[i] > match_threshold
fpmask = tf.cast(pmask, dtype)
n_positives = tf.reduce_sum(fpmask) # Select some random negative entries.
# n_entries = np.prod(gclasses[i].get_shape().as_list())
# r_positive = n_positives / n_entries
# r_negative = negative_ratio * n_positives / (n_entries - n_positives) # Negative mask.
no_classes = tf.cast(pmask, tf.int32)
predictions = slim.softmax(logits[i])
nmask = tf.logical_and(tf.logical_not(pmask),
gscores[i] > -0.5)
fnmask = tf.cast(nmask, dtype)
nvalues = tf.where(nmask,
predictions[:, :, :, :, 0],
1. - fnmask)
nvalues_flat = tf.reshape(nvalues, [-1])
# Number of negative entries to select.
n_neg = tf.cast(negative_ratio * n_positives, tf.int32)
n_neg = tf.maximum(n_neg, tf.size(nvalues_flat) // 8)
n_neg = tf.maximum(n_neg, tf.shape(nvalues)[0] * 4)
max_neg_entries = 1 + tf.cast(tf.reduce_sum(fnmask), tf.int32)
n_neg = tf.minimum(n_neg, max_neg_entries) val, idxes = tf.nn.top_k(-nvalues_flat, k=n_neg)
max_hard_pred = -val[-1]
# Final negative mask.
nmask = tf.logical_and(nmask, nvalues < max_hard_pred)
fnmask = tf.cast(nmask, dtype) # Add cross-entropy loss.
with tf.name_scope('cross_entropy_pos'):
fpmask = wsize * fpmask
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits[i],
labels=gclasses[i])
loss = tf.losses.compute_weighted_loss(loss, fpmask)
l_cross_pos.append(loss) with tf.name_scope('cross_entropy_neg'):
fnmask = wsize * fnmask
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits[i],
labels=no_classes)
loss = tf.losses.compute_weighted_loss(loss, fnmask)
l_cross_neg.append(loss) # Add localization loss: smooth L1, L2, ...
with tf.name_scope('localization'):
# Weights Tensor: positive mask + random negative.
weights = tf.expand_dims(alpha * fpmask, axis=-1)
loss = custom_layers.abs_smooth(localisations[i] - glocalisations[i])
loss = tf.losses.compute_weighted_loss(loss, weights)
l_loc.append(loss) # Additional total losses...
with tf.name_scope('total'):
total_cross_pos = tf.add_n(l_cross_pos, 'cross_entropy_pos')
total_cross_neg = tf.add_n(l_cross_neg, 'cross_entropy_neg')
total_cross = tf.add(total_cross_pos, total_cross_neg, 'cross_entropy')
total_loc = tf.add_n(l_loc, 'localization') # Add to EXTRA LOSSES TF.collection
tf.add_to_collection('EXTRA_LOSSES', total_cross_pos)
tf.add_to_collection('EXTRA_LOSSES', total_cross_neg)
tf.add_to_collection('EXTRA_LOSSES', total_cross)
tf.add_to_collection('EXTRA_LOSSES', total_loc)

# Copyright 2015 Paul Balanca. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Implement some custom layers, not provided by TensorFlow.
实现一些TensorFlow没有提供的自定义层
Trying to follow as much as possible the style/standards used in
tf.contrib.layers
尽可能多地遵循这种风格/标准
"""
import tensorflow as tf from tensorflow.contrib.framework.python.ops import add_arg_scope
from tensorflow.contrib.layers.python.layers import initializers
from tensorflow.contrib.framework.python.ops import variables
from tensorflow.contrib.layers.python.layers import utils
from tensorflow.python.ops import nn
from tensorflow.python.ops import init_ops
from tensorflow.python.ops import variable_scope def abs_smooth(x):
"""Smoothed absolute function. Useful to compute an L1 smooth error.
#绝对平滑函数，用于计算L1平滑误差
#当预测值与目标值相差很大时, 梯度容易爆炸,因此L1 loss对噪声（outliers）更鲁棒
Define as:
x^2 / 2 if abs(x) < 1
abs(x) - 0.5 if abs(x) > 1
We use here a differentiable definition using min(x) and abs(x). Clearly
not optimal, but good enough for our purpose!
"""
absx = tf.abs(x)
minx = tf.minimum(absx, 1)
r = 0.5 * ((absx - 1) * minx + absx) #计算得到L1 smooth loss
return r @add_arg_scope
#L2正则化：稀疏正则化操作
def l2_normalization(
inputs,#输入特征层，[batch_size,h,w,c]
scaling=False,#默认归一化后是否设置缩放变量gamma
scale_initializer=init_ops.ones_initializer(),#scale初始化为1
reuse=None,
variables_collections=None,
outputs_collections=None,
data_format='NHWC',
trainable=True,
scope=None):
"""Implement L2 normalization on every feature (i.e. spatial normalization).
对每个特性实现L2规范化，空间归一化
Should be extended in some near future to other dimensions, providing a more
flexible normalization framework.
是否应该在不久的将来扩展到其他维度，提供更多灵活的标准化框架。
Args:
inputs: a 4-D tensor with dimensions [batch_size, height, width, channels].
scaling: whether or not to add a post scaling operation along the dimensions
which have been normalized.
scale_initializer: An initializer for the weights.
reuse: whether or not the layer and its variables should be reused. To be
able to reuse the layer scope must be given.
variables_collections: optional list of collections for all the variables or
a dictionary containing a different list of collection per variable.
outputs_collections: collection to add the outputs.
data_format: NHWC or NCHW data format.
trainable: If `True` also add variables to the graph collection
`GraphKeys.TRAINABLE_VARIABLES` (see tf.Variable).
scope: Optional scope for `variable_scope`.
Returns:
A `Tensor` representing the output of the operation.
""" with variable_scope.variable_scope(
scope, 'L2Normalization', [inputs], reuse=reuse) as sc:
inputs_shape = inputs.get_shape()#得到输入特征层的维度信息
inputs_rank = inputs_shape.ndims #维度数=4
dtype = inputs.dtype.base_dtype#数据类型
if data_format == 'NHWC':
# norm_dim = tf.range(1, inputs_rank-1)
norm_dim = tf.range(inputs_rank-1, inputs_rank)#需要正则化的维度是4-1=3即channel这个维度
params_shape = inputs_shape[-1:]#通道数
elif data_format == 'NCHW':
# norm_dim = tf.range(2, inputs_rank)
norm_dim = tf.range(1, 2)#需要正则化的维度是第1维，即channel这个维度
params_shape = (inputs_shape[1])#通道数 # Normalize along spatial dimensions.
# 对通道所在维度进行正则化，其中epsilon是避免除0风险
outputs = nn.l2_normalize(inputs, norm_dim, epsilon=1e-12)
# Additional scaling.
# 判断是否对正则化后设置缩放变量
if scaling:
scale_collections = utils.get_variable_collections(
variables_collections, 'scale')
scale = variables.model_variable('gamma',
shape=params_shape,
dtype=dtype,
initializer=scale_initializer,
collections=scale_collections,
trainable=trainable)
if data_format == 'NHWC':
outputs = tf.multiply(outputs, scale)
elif data_format == 'NCHW':
scale = tf.expand_dims(scale, axis=-1)
scale = tf.expand_dims(scale, axis=-1)
outputs = tf.multiply(outputs, scale)
# outputs = tf.transpose(outputs, perm=(0, 2, 3, 1))
# 即返回L2_norm*gamma
return utils.collect_named_outputs(outputs_collections,
sc.original_name_scope, outputs) @add_arg_scope
def pad2d(inputs,
pad=(0, 0),
mode='CONSTANT',
data_format='NHWC',
trainable=True,
scope=None):
"""
2D Padding layer, adding a symmetric padding to H and W dimensions.
2D填充层，为H和W维度添加对称填充
Aims to mimic padding in Caffe and MXNet, helping the port of models to
TensorFlow. Tries to follow the naming convention of `tf.contrib.layers`.
目的是在Caffe和MXNet中模拟填充，帮助模型移植到TensorFlow。
尝试遵循“tf.contrib.layers”的命名约定。
Args:
inputs: 4D input Tensor;
pad: 2-Tuple with padding values for H and W dimensions;
mode: Padding mode. C.f. `tf.pad`
data_format: NHWC or NCHW data format.
"""
with tf.name_scope(scope, 'pad2d', [inputs]):
# Padding shape.
if data_format == 'NHWC':
paddings = [[0, 0], [pad[0], pad[0]], [pad[1], pad[1]], [0, 0]]
elif data_format == 'NCHW':
paddings = [[0, 0], [0, 0], [pad[0], pad[0]], [pad[1], pad[1]]]
net = tf.pad(inputs, paddings, mode=mode)
return net @add_arg_scope
#作用，将输入的特征图网络的通道维度放在最后，返回变形后的网络
def channel_to_last(inputs,
data_format='NHWC',
scope=None):
"""Move the channel axis to the last dimension. Allows to
provide a single output format whatever the input data format.
将通道轴移动到最后一个维度。允许无论输入数据格式如何，都要提供单一的输出格式。
Args:
inputs: Input Tensor;
data_format: NHWC or NCHW.
Return:
Input in NHWC format.
"""
with tf.name_scope(scope, 'channel_to_last', [inputs]):
if data_format == 'NHWC':
net = inputs
elif data_format == 'NCHW':
net = tf.transpose(inputs, perm=(0, 2, 3, 1))
return net

# Copyright 2015 Paul Balanca. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Shared function between different SSD implementations.
"""
import numpy as np
import tensorflow as tf
import tf_extended as tfe # =========================================================================== #
# TensorFlow implementation of boxes SSD encoding / decoding.
# =========================================================================== #
def tf_ssd_bboxes_encode_layer(labels, #gt标签，1D的tensor
bboxes, #Nx4的Tensor(float)，真实的bbox
anchors_layer, #参考锚点list
num_classes, #分类类别数
no_annotation_label,
ignore_threshold=0.5, #gt和锚点框间的匹配阈值，大于该值则为正样本
prior_scaling=[0.1, 0.1, 0.2, 0.2], #真实值到预测值转换中用到的缩放
dtype=tf.float32):
"""Encode groundtruth labels and bounding boxes using SSD anchors from
one layer.
Arguments:
labels: 1D Tensor(int64) containing groundtruth labels;
bboxes: Nx4 Tensor(float) with bboxes relative coordinates;
anchors_layer: Numpy array with layer anchors;
matching_threshold: Threshold for positive match with groundtruth bboxes;
prior_scaling: Scaling of encoded coordinates.
Return:
(target_labels, target_localizations, target_scores): Target Tensors. 返回：包含目标标签类别，目标位置，目标置信度的tesndor
"""
# Anchors coordinates and volume.
yref, xref, href, wref = anchors_layer #此前每个特征图上点对应生成的锚点框作为参考框
ymin = yref - href / 2. #求参考框的左上角点（xmin,ymin）和右下角点(xmax,ymax)
xmin = xref - wref / 2. #yref和xref的shape为（38,38,1）；href和wref的shape为（4，）
ymax = yref + href / 2.
xmax = xref + wref / 2.
vol_anchors = (xmax - xmin) * (ymax - ymin) #求参考框面积vol_anchors # Initialize tensors... #shape表示每个特征图上总锚点数
shape = (yref.shape[0], yref.shape[1], href.size) #对于第一个特征图，shape=(38,38,4)；第二个特征图的shape=(19,19,6)
feat_labels = tf.zeros(shape, dtype=tf.int64) #初始化每个特征图上的点对应的各个box所属标签维度 如：38x38x4
feat_scores = tf.zeros(shape, dtype=dtype) #初始化每个特征图上的点对应的各个box所属标目标的得分值维度 如：38x38x4 feat_ymin = tf.zeros(shape, dtype=dtype) #预测每个特征图每个点所属目标的坐标 ；如38x38x4;初始化为全0
feat_xmin = tf.zeros(shape, dtype=dtype)
feat_ymax = tf.ones(shape, dtype=dtype)
feat_xmax = tf.ones(shape, dtype=dtype) def jaccard_with_anchors(bbox): #计算gt的框和参考锚点框的重合度
"""Compute jaccard score between a box and the anchors.
"""
int_ymin = tf.maximum(ymin, bbox[0]) #计算重叠区域的坐标
int_xmin = tf.maximum(xmin, bbox[1])
int_ymax = tf.minimum(ymax, bbox[2])
int_xmax = tf.minimum(xmax, bbox[3])
h = tf.maximum(int_ymax - int_ymin, 0.) #计算重叠区域的长与宽
w = tf.maximum(int_xmax - int_xmin, 0.)
# Volumes.
inter_vol = h * w #重叠区域的面积
union_vol = vol_anchors - inter_vol \ #计算bbox和参考框的并集区域
+ (bbox[2] - bbox[0]) * (bbox[3] - bbox[1])
jaccard = tf.div(inter_vol, union_vol) #计算IOU并返回该值
return jaccard def intersection_with_anchors(bbox): #计算某个参考框包含真实框的得分情况
"""Compute intersection between score a box and the anchors.
"""
int_ymin = tf.maximum(ymin, bbox[0]) #计算bbox和锚点框重叠区域的坐标和长宽
int_xmin = tf.maximum(xmin, bbox[1])
int_ymax = tf.minimum(ymax, bbox[2])
int_xmax = tf.minimum(xmax, bbox[3])
h = tf.maximum(int_ymax - int_ymin, 0.)
w = tf.maximum(int_xmax - int_xmin, 0.)
inter_vol = h * w #重叠区域面积
scores = tf.div(inter_vol, vol_anchors) #将重叠区域面积除以参考框面积作为该参考框得分值；
return scores def condition(i, feat_labels, feat_scores,
feat_ymin, feat_xmin, feat_ymax, feat_xmax):
"""Condition: check label index.
"""
r = tf.less(i, tf.shape(labels)) # 逐元素比较大小,遍历labels，因为i在body返回的时候加1了
return r[0] def body(i, feat_labels, feat_scores, #该函数大致意思是选择与gt box IOU最大的锚点框负责回归任务，并预测对应的边界框，如此循环
feat_ymin, feat_xmin, feat_ymax, feat_xmax):
"""Body: update feature labels, scores and bboxes.
Follow the original SSD paper for that purpose:
- assign values when jaccard > 0.5;
- only update if beat the score of other bboxes.
"""
# Jaccard score. #计算bbox与参考框的IOU值
label = labels[i]
bbox = bboxes[i]
jaccard = jaccard_with_anchors(bbox)
# Mask: check threshold + scores + no annotations + num_classes.
mask = tf.greater(jaccard, feat_scores) #当IOU大于feat_scores时，对应的mask至1，做筛选
# mask = tf.logical_and(mask, tf.greater(jaccard, matching_threshold))
mask = tf.logical_and(mask, feat_scores > -0.5)
mask = tf.logical_and(mask, label < num_classes) #label满足<21
imask = tf.cast(mask, tf.int64) #将mask转换数据类型int型
fmask = tf.cast(mask, dtype) #将mask转换数据类型float型
# Update values using mask.
feat_labels = imask * label + (1 - imask) * feat_labels #当mask=1，则feat_labels=1；否则为0，即背景
feat_scores = tf.where(mask, jaccard, feat_scores) #tf.where表示如果mask为真则jaccard，否则为feat_scores feat_ymin = fmask * bbox[0] + (1 - fmask) * feat_ymin #选择与GT bbox IOU最大的框作为GT bbox，然后循环
feat_xmin = fmask * bbox[1] + (1 - fmask) * feat_xmin
feat_ymax = fmask * bbox[2] + (1 - fmask) * feat_ymax
feat_xmax = fmask * bbox[3] + (1 - fmask) * feat_xmax # Check no annotation label: ignore these anchors... #对没有标注标签的锚点框做忽视，应该是背景
# interscts = intersection_with_anchors(bbox)
# mask = tf.logical_and(interscts > ignore_threshold,
# label == no_annotation_label)
# # Replace scores by -1.
# feat_scores = tf.where(mask, -tf.cast(mask, dtype), feat_scores) return [i+1, feat_labels, feat_scores,
feat_ymin, feat_xmin, feat_ymax, feat_xmax]
# Main loop definition.
i = 0
[i, feat_labels, feat_scores,
feat_ymin, feat_xmin,
feat_ymax, feat_xmax] = tf.while_loop(condition, body,
[i, feat_labels, feat_scores,
feat_ymin, feat_xmin,
feat_ymax, feat_xmax])
# Transform to center / size. #转换为中心及长宽形式（计算补偿后的中心）
feat_cy = (feat_ymax + feat_ymin) / 2. #真实预测值其实是边界框相对于先验框的转换值，encode就是为了求这个转换值
feat_cx = (feat_xmax + feat_xmin) / 2.
feat_h = feat_ymax - feat_ymin
feat_w = feat_xmax - feat_xmin
# Encode features.
feat_cy = (feat_cy - yref) / href / prior_scaling[0] #(预测真实边界框中心y-参考框中心y)/参考框高/缩放尺度
feat_cx = (feat_cx - xref) / wref / prior_scaling[1]
feat_h = tf.log(feat_h / href) / prior_scaling[2] #log(预测真实边界框高h/参考框高h)/缩放尺度
feat_w = tf.log(feat_w / wref) / prior_scaling[3]
# Use SSD ordering: x / y / w / h instead of ours.
feat_localizations = tf.stack([feat_cx, feat_cy, feat_w, feat_h], axis=-1) #返回（cx转换值,cy转换值,w转换值,h转换值）形式的边界框的预测值（其实是预测框相对于参考框的转换）
return feat_labels, feat_localizations, feat_scores #返回目标标签，目标预测值（位置转换值），目标置信度
#经过我们回归得到的变换，经过变换得到真实框，所以这个地方损失函数其实是我们预测的是变换，我们实际的框和anchor之间的变换和我们预测的变换之间的loss。我们回归的是一种变换。并不是直接预测框，这个和YOLO是不一样的。和Faster RCNN是一样的 def tf_ssd_bboxes_encode(labels, #1D的tensor 包含gt标签
bboxes, #Nx4的tensor包含真实框的相对坐标
anchors, #参考锚点框信息（y,x,h,w) 其中y,x是中心坐标
num_classes,
no_annotation_label,
ignore_threshold=0.5,
prior_scaling=[0.1, 0.1, 0.2, 0.2],
dtype=tf.float32,
scope='ssd_bboxes_encode'):
"""Encode groundtruth labels and bounding boxes using SSD net anchors.
Encoding boxes for all feature layers.
Arguments:
labels: 1D Tensor(int64) containing groundtruth labels;
bboxes: Nx4 Tensor(float) with bboxes relative coordinates;
anchors: List of Numpy array with layer anchors;
matching_threshold: Threshold for positive match with groundtruth bboxes;
prior_scaling: Scaling of encoded coordinates.
Return:
(target_labels, target_localizations, target_scores): #返回：目标标签，目标位置，目标得分值（都是list形式）
Each element is a list of target Tensors.
"""
with tf.name_scope(scope):
target_labels = [] #目标标签
target_localizations = [] #目标位置
target_scores = [] #目标得分
for i, anchors_layer in enumerate(anchors): #对所有特征图中的参考框做遍历
with tf.name_scope('bboxes_encode_block_%i' % i):
t_labels, t_loc, t_scores = \
tf_ssd_bboxes_encode_layer(labels, bboxes, anchors_layer, #输入真实标签，gt位置大小，参考框位置大小……得到预测真实标签，参考框到真实框的转换以及得分
num_classes, no_annotation_label,
ignore_threshold,
prior_scaling, dtype)
target_labels.append(t_labels)
target_localizations.append(t_loc)
target_scores.append(t_scores)
return target_labels, target_localizations, target_scores def tf_ssd_bboxes_decode_layer(feat_localizations, #解码，在预测时用到，根据之前得到的预测值相对于参考框的转换值后，反推出真实位置（该位置包括真实的x,y,w,h）
anchors_layer, #需要输入：预测框和参考框的转换feat_localizations，参考框位置尺度信息anchors_layer，以及转换时用到的缩放
prior_scaling=[0.1, 0.1, 0.2, 0.2]): #输出真实预测框的ymin,xmin，ymax,xmax
"""Compute the relative bounding boxes from the layer features and
reference anchor bounding boxes.
Arguments:
feat_localizations: Tensor containing localization features.
anchors: List of numpy array containing anchor boxes.
Return:
Tensor Nx4: ymin, xmin, ymax, xmax
"""
yref, xref, href, wref = anchors_layer #锚点框的参考中心点以及长宽 # Compute center, height and width
cx = feat_localizations[:, :, :, :, 0] * wref * prior_scaling[0] + xref
cy = feat_localizations[:, :, :, :, 1] * href * prior_scaling[1] + yref
w = wref * tf.exp(feat_localizations[:, :, :, :, 2] * prior_scaling[2])
h = href * tf.exp(feat_localizations[:, :, :, :, 3] * prior_scaling[3])
# Boxes coordinates.
ymin = cy - h / 2.
xmin = cx - w / 2.
ymax = cy + h / 2.
xmax = cx + w / 2.
bboxes = tf.stack([ymin, xmin, ymax, xmax], axis=-1)
return bboxes #预测真实框的坐标信息（两点式的框） def tf_ssd_bboxes_decode(feat_localizations,
anchors,
prior_scaling=[0.1, 0.1, 0.2, 0.2],
scope='ssd_bboxes_decode'):
"""Compute the relative bounding boxes from the SSD net features and
reference anchors bounding boxes.
Arguments:
feat_localizations: List of Tensors containing localization features.
anchors: List of numpy array containing anchor boxes.
Return:
List of Tensors Nx4: ymin, xmin, ymax, xmax
"""
with tf.name_scope(scope):
bboxes = []
for i, anchors_layer in enumerate(anchors):
bboxes.append(
tf_ssd_bboxes_decode_layer(feat_localizations[i],
anchors_layer,
prior_scaling))
return bboxes # =========================================================================== #
# SSD boxes selection.
# =========================================================================== #
def tf_ssd_bboxes_select_layer(predictions_layer, localizations_layer, #输入预测得到的类别和位置做筛选
select_threshold=None,
num_classes=21,
ignore_class=0,
scope=None):
"""Extract classes, scores and bounding boxes from features in one layer.
Batch-compatible: inputs are supposed to have batch-type shapes.
Args:
predictions_layer: A SSD prediction layer;
localizations_layer: A SSD localization layer;
select_threshold: Classification threshold for selecting a box. All boxes
under the threshold are set to 'zero'. If None, no threshold applied.
Return:
d_scores, d_bboxes: Dictionary of scores and bboxes Tensors of
size Batches X N x 1 | 4. Each key corresponding to a class.
"""
select_threshold = 0.0 if select_threshold is None else select_threshold
with tf.name_scope(scope, 'ssd_bboxes_select_layer',
[predictions_layer, localizations_layer]):
# Reshape features: Batches x N x N_labels | 4
p_shape = tfe.get_shape(predictions_layer)
predictions_layer = tf.reshape(predictions_layer,
tf.stack([p_shape[0], -1, p_shape[-1]]))
l_shape = tfe.get_shape(localizations_layer)
localizations_layer = tf.reshape(localizations_layer,
tf.stack([l_shape[0], -1, l_shape[-1]])) d_scores = {}
d_bboxes = {}
for c in range(0, num_classes):
if c != ignore_class: #如果不是背景类别
# Remove boxes under the threshold. #去掉低于阈值的box
scores = predictions_layer[:, :, c] #预测为第c类别的得分值
fmask = tf.cast(tf.greater_equal(scores, select_threshold), scores.dtype)
scores = scores * fmask #保留得分值大于阈值的得分
bboxes = localizations_layer * tf.expand_dims(fmask, axis=-1)
# Append to dictionary.
d_scores[c] = scores
d_bboxes[c] = bboxes return d_scores, d_bboxes #返回字典，每个字典里是对应某类的预测权重和框位置信息； def tf_ssd_bboxes_select(predictions_net, localizations_net, #输入：SSD网络输出的预测层list；定位层list；类别选择框阈值（None表示都选）
select_threshold=None, #返回一个字典，key为类别，值为得分和bbox坐标
num_classes=21, #包含了背景类别
ignore_class=0, #第0类是背景
scope=None):
"""Extract classes, scores and bounding boxes from network output layers.
Batch-compatible: inputs are supposed to have batch-type shapes.
Args:
predictions_net: List of SSD prediction layers;
localizations_net: List of localization layers;
select_threshold: Classification threshold for selecting a box. All boxes
under the threshold are set to 'zero'. If None, no threshold applied.
Return:
d_scores, d_bboxes: Dictionary of scores and bboxes Tensors of #返回一个字典，其中key是对应类别，值对应得分值和坐标信息
size Batches X N x 1 | 4. Each key corresponding to a class.
"""
with tf.name_scope(scope, 'ssd_bboxes_select',
[predictions_net, localizations_net]):
l_scores = []
l_bboxes = []
for i in range(len(predictions_net)):
scores, bboxes = tf_ssd_bboxes_select_layer(predictions_net[i],
localizations_net[i],
select_threshold,
num_classes,
ignore_class)
l_scores.append(scores) #对应某个类别的得分
l_bboxes.append(bboxes) #对应某个类别的box坐标信息
# Concat results.
d_scores = {}
d_bboxes = {}
for c in l_scores[0].keys():
ls = [s[c] for s in l_scores]
lb = [b[c] for b in l_bboxes]
d_scores[c] = tf.concat(ls, axis=1)
d_bboxes[c] = tf.concat(lb, axis=1)
return d_scores, d_bboxes def tf_ssd_bboxes_select_layer_all_classes(predictions_layer, localizations_layer,
select_threshold=None):
"""Extract classes, scores and bounding boxes from features in one layer.
Batch-compatible: inputs are supposed to have batch-type shapes.
Args:
predictions_layer: A SSD prediction layer;
localizations_layer: A SSD localization layer;
select_threshold: Classification threshold for selecting a box. If None,
select boxes whose classification score is higher than 'no class'.
Return:
classes, scores, bboxes: Input Tensors. #输出：类别，得分，框
"""
# Reshape features: Batches x N x N_labels | 4
p_shape = tfe.get_shape(predictions_layer)
predictions_layer = tf.reshape(predictions_layer,
tf.stack([p_shape[0], -1, p_shape[-1]]))
l_shape = tfe.get_shape(localizations_layer)
localizations_layer = tf.reshape(localizations_layer,
tf.stack([l_shape[0], -1, l_shape[-1]]))
# Boxes selection: use threshold or score > no-label criteria.
if select_threshold is None or select_threshold == 0:
# Class prediction and scores: assign 0. to 0-class
classes = tf.argmax(predictions_layer, axis=2)
scores = tf.reduce_max(predictions_layer, axis=2)
scores = scores * tf.cast(classes > 0, scores.dtype)
else:
sub_predictions = predictions_layer[:, :, 1:]
classes = tf.argmax(sub_predictions, axis=2) + 1
scores = tf.reduce_max(sub_predictions, axis=2)
# Only keep predictions higher than threshold.
mask = tf.greater(scores, select_threshold)
classes = classes * tf.cast(mask, classes.dtype)
scores = scores * tf.cast(mask, scores.dtype)
# Assume localization layer already decoded.
bboxes = localizations_layer
return classes, scores, bboxes #寻找当前特征图中类别，得分，bbox def tf_ssd_bboxes_select_all_classes(predictions_net, localizations_net,
select_threshold=None,
scope=None):
"""Extract classes, scores and bounding boxes from network output layers.
Batch-compatible: inputs are supposed to have batch-type shapes.
Args:
predictions_net: List of SSD prediction layers;
localizations_net: List of localization layers;
select_threshold: Classification threshold for selecting a box. If None,
select boxes whose classification score is higher than 'no class'.
Return:
classes, scores, bboxes: Tensors.
"""
with tf.name_scope(scope, 'ssd_bboxes_select',
[predictions_net, localizations_net]):
l_classes = []
l_scores = []
l_bboxes = []
for i in range(len(predictions_net)):
classes, scores, bboxes = \
tf_ssd_bboxes_select_layer_all_classes(predictions_net[i],
localizations_net[i],
select_threshold)
l_classes.append(classes)
l_scores.append(scores)
l_bboxes.append(bboxes) classes = tf.concat(l_classes, axis=1)
scores = tf.concat(l_scores, axis=1)
bboxes = tf.concat(l_bboxes, axis=1)
return classes, scores, bboxes #返回所有特征图综合得出的类别，得分，bbox