diff --git a/python-package/insightface/__init__.py b/python-package/insightface/__init__.py index fb3068b..21df290 100644 --- a/python-package/insightface/__init__.py +++ b/python-package/insightface/__init__.py @@ -20,9 +20,9 @@ except ImportError: "Unable to import dependency mxnet. " "A quick tip is to install via `pip install mxnet-mkl/mxnet-cu90mkl --pre`. ") -__version__ = '0.1.2' +__version__ = '0.1.3' from . import model_zoo -#from . import utils -#from . import analysis +from . import utils +from . import app diff --git a/python-package/insightface/app/__init__.py b/python-package/insightface/app/__init__.py new file mode 100644 index 0000000..cc57461 --- /dev/null +++ b/python-package/insightface/app/__init__.py @@ -0,0 +1 @@ +from .face_analysis import * diff --git a/python-package/insightface/app/face_analysis.py b/python-package/insightface/app/face_analysis.py new file mode 100644 index 0000000..69be3f1 --- /dev/null +++ b/python-package/insightface/app/face_analysis.py @@ -0,0 +1,72 @@ +from __future__ import division +import collections +import mxnet as mx +import numpy as np +from numpy.linalg import norm +import mxnet.ndarray as nd +from ..model_zoo import model_zoo +from ..utils import face_align + +__all__ = ['FaceAnalysis', + 'Face'] + +Face = collections.namedtuple('Face', [ + 'bbox', 'landmark', 'det_score', 'embedding', 'gender', 'age', 'embedding_norm', 'normed_embedding']) + +Face.__new__.__defaults__ = (None,) * len(Face._fields) + +class FaceAnalysis: + def __init__(self, det_name='retinaface_r50_v1', rec_name='arcface_r100_v1', ga_name='genderage_v1'): + assert det_name is not None + self.det_model = model_zoo.get_model(det_name) + if rec_name is not None: + self.rec_model = model_zoo.get_model(rec_name) + else: + self.rec_model = None + if ga_name is not None: + self.ga_model = model_zoo.get_model(ga_name) + else: + self.ga_model = None + + def prepare(self, ctx_id, nms=0.4): + self.det_model.prepare(ctx_id, nms) + if self.rec_model is not None: + self.rec_model.prepare(ctx_id) + if self.ga_model is not None: + self.ga_model.prepare(ctx_id) + + def get(self, img, det_thresh = 0.8, det_scale = 1.0, max_num = 0): + bboxes, landmarks = self.det_model.detect(img, threshold=det_thresh, scale = det_scale) + if bboxes.shape[0]==0: + return [] + if max_num>0 and bboxes.shape[0]>max_num: + area = (bboxes[:,2]-bboxes[:,0])*(bboxes[:,3]-bboxes[:,1]) + img_center = img.shape[0]//2, img.shape[1]//2 + offsets = np.vstack([ (bboxes[:,0]+bboxes[:,2])/2-img_center[1], (bboxes[:,1]+bboxes[:,3])/2-img_center[0] ]) + offset_dist_squared = np.sum(np.power(offsets,2.0),0) + bindex = np.argmax(area-offset_dist_squared*2.0) # some extra weight on the centering + bindex = bindex[0:max_num] + bboxes = bboxes[bindex, :] + landmarks = landmarks[bindex, :] + ret = [] + for i in range(bboxes.shape[0]): + bbox = bboxes[i, 0:4] + det_score = bboxes[i,4] + landmark = landmarks[i] + _img = face_align.norm_crop(img, landmark = landmark) + embedding = None + embedding_norm = None + normed_embedding = None + gender = None + age = None + if self.rec_model is not None: + embedding = self.rec_model.get_embedding(_img).flatten() + embedding_norm = norm(embedding) + normed_embedding = embedding / embedding_norm + if self.ga_model is not None: + gender, age = self.ga_model.get(_img) + face = Face(bbox = bbox, landmark = landmark, det_score = det_score, embedding = embedding, gender = gender, age = age + , normed_embedding=normed_embedding, embedding_norm = embedding_norm) + ret.append(face) + return ret + diff --git a/python-package/insightface/model_zoo/face_detection.py b/python-package/insightface/model_zoo/face_detection.py index 84f7a3e..41edcab 100644 --- a/python-package/insightface/model_zoo/face_detection.py +++ b/python-package/insightface/model_zoo/face_detection.py @@ -6,6 +6,7 @@ import mxnet.ndarray as nd __all__ = ['FaceDetector', 'retinaface_r50_v1', 'retinaface_mnet025_v1', + 'retinaface_mnet025_v2', 'get_retinaface'] def _whctrs(anchor): @@ -222,10 +223,14 @@ class FaceDetector: self.model = model self.nms_threshold = nms + self.landmark_std = 1.0 _ratio = (1.,) fmc = 3 if self.rac=='net3': _ratio = (1.,) + elif self.rac=='net3l': + _ratio = (1.,) + self.landmark_std = 0.2 elif network=='net5': #retinaface fmc = 5 else: @@ -268,8 +273,6 @@ class FaceDetector: v = self._anchors_fpn[k].astype(np.float32) self._anchors_fpn[k] = v self.anchor_plane_cache = {} - if fix_image_size is None: - self.anchor_plane_cache = None self._num_anchors = dict(zip(self.fpn_keys, [anchors.shape[0] for anchors in self._anchors_fpn.values()])) @@ -304,19 +307,15 @@ class FaceDetector: height, width = bbox_deltas.shape[2], bbox_deltas.shape[3] A = self._num_anchors['stride%s'%s] K = height * width - if self.anchor_plane_cache is not None: - key = (height, width, stride) - if key in self.anchor_plane_cache: - anchors = self.anchor_plane_cache[key] - else: - anchors_fpn = self._anchors_fpn['stride%s'%s] - anchors = anchors_plane(height, width, stride, anchors_fpn) - anchors = anchors.reshape((K * A, 4)) - self.anchor_plane_cache[key] = anchors + key = (height, width, stride) + if key in self.anchor_plane_cache: + anchors = self.anchor_plane_cache[key] else: anchors_fpn = self._anchors_fpn['stride%s'%s] anchors = anchors_plane(height, width, stride, anchors_fpn) anchors = anchors.reshape((K * A, 4)) + if len(self.anchor_plane_cache)<100: + self.anchor_plane_cache[key] = anchors scores = clip_pad(scores, (height, width)) scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1)) @@ -346,6 +345,7 @@ class FaceDetector: landmark_deltas = clip_pad(landmark_deltas, (height, width)) landmark_pred_len = landmark_deltas.shape[1]//A landmark_deltas = landmark_deltas.transpose((0, 2, 3, 1)).reshape((-1, 5, landmark_pred_len//5)) + landmark_deltas *= self.landmark_std #print(landmark_deltas.shape, landmark_deltas) landmarks = landmark_pred(anchors, landmark_deltas) landmarks = landmarks[order, :] @@ -420,3 +420,6 @@ def retinaface_r50_v1(**kwargs): def retinaface_mnet025_v1(**kwargs): return get_retinaface("mnet025_v1", rac='net3', **kwargs) +def retinaface_mnet025_v2(**kwargs): + return get_retinaface("mnet025_v2", rac='net3l', **kwargs) + diff --git a/python-package/insightface/model_zoo/face_genderage.py b/python-package/insightface/model_zoo/face_genderage.py new file mode 100644 index 0000000..3a0423d --- /dev/null +++ b/python-package/insightface/model_zoo/face_genderage.py @@ -0,0 +1,77 @@ +from __future__ import division +import mxnet as mx +import numpy as np +import cv2 + +__all__ = ['FaceGenderage', + 'genderage_v1', + 'get_genderage'] + + +class FaceGenderage: + def __init__(self, name, download, param_file): + self.name = name + self.download = download + self.param_file = param_file + self.image_size = (112, 112) + if download: + assert param_file + + def prepare(self, ctx_id): + if self.param_file: + pos = self.param_file.rfind('-') + prefix = self.param_file[0:pos] + pos2 = self.param_file.rfind('.') + epoch = int(self.param_file[pos+1:pos2]) + sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) + all_layers = sym.get_internals() + sym = all_layers['fc1_output'] + if ctx_id>=0: + ctx = mx.gpu(ctx_id) + else: + ctx = mx.cpu() + model = mx.mod.Module(symbol=sym, context=ctx, label_names = None) + data_shape = (1,3)+self.image_size + model.bind(data_shapes=[('data', data_shape)]) + model.set_params(arg_params, aux_params) + #warmup + data = mx.nd.zeros(shape=data_shape) + db = mx.io.DataBatch(data=(data,)) + model.forward(db, is_train=False) + embedding = model.get_outputs()[0].asnumpy() + self.model = model + else: + pass + + def get(self, img): + assert self.param_file and self.model + assert img.shape[2]==3 and img.shape[0:2]==self.image_size + data = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) + data = np.transpose(data, (2,0,1)) + data = np.expand_dims(data, axis=0) + data = mx.nd.array(data) + db = mx.io.DataBatch(data=(data,)) + self.model.forward(db, is_train=False) + ret = self.model.get_outputs()[0].asnumpy() + g = ret[:,0:2].flatten() + gender = np.argmax(g) + a = ret[:,2:202].reshape( (100,2) ) + a = np.argmax(a, axis=1) + age = int(sum(a)) + return gender, age + +def get_genderage(name, download=True, + root='~/.insightface/models', **kwargs): + if not download: + return FaceGenderage(name, False, None) + else: + from .model_store import get_model_file + _file = get_model_file("genderage_%s"%name, root=root) + return FaceGenderage(name, True, _file) + +def genderage_v1(**kwargs): + return get_genderage("v1", download=True, **kwargs) + + + + diff --git a/python-package/insightface/model_zoo/model_store.py b/python-package/insightface/model_zoo/model_store.py index 87e0388..e76eafb 100644 --- a/python-package/insightface/model_zoo/model_store.py +++ b/python-package/insightface/model_zoo/model_store.py @@ -16,6 +16,8 @@ _model_sha1 = {name: checksum for checksum, name in [ ('', 'arcface_mfn_v1'), ('39fd1e087a2a2ed70a154ac01fecaa86c315d01b', 'retinaface_r50_v1'), ('2c9de8116d1f448fd1d4661f90308faae34c990a', 'retinaface_mnet025_v1'), + ('0db1d07921d005e6c9a5b38e059452fc5645e5a4', 'retinaface_mnet025_v2'), + ('7dd8111652b7aac2490c5dcddeb268e53ac643e6', 'genderage_v1'), ]} base_repo_url = 'http://insightface.ai/files/' diff --git a/python-package/insightface/model_zoo/model_zoo.py b/python-package/insightface/model_zoo/model_zoo.py index 03b9801..c3298ad 100644 --- a/python-package/insightface/model_zoo/model_zoo.py +++ b/python-package/insightface/model_zoo/model_zoo.py @@ -4,6 +4,7 @@ This code file mainly comes from https://github.com/dmlc/gluon-cv/blob/master/gl """ from .face_recognition import * from .face_detection import * +from .face_genderage import * #from .face_alignment import * __all__ = ['get_model', 'get_model_list'] @@ -14,6 +15,8 @@ _models = { #'arcface_outofreach_v1': arcface_outofreach_v1, 'retinaface_r50_v1': retinaface_r50_v1, 'retinaface_mnet025_v1': retinaface_mnet025_v1, + 'retinaface_mnet025_v2': retinaface_mnet025_v2, + 'genderage_v1': genderage_v1, } diff --git a/python-package/insightface/utils/face_align.py b/python-package/insightface/utils/face_align.py new file mode 100644 index 0000000..6cf5a2e --- /dev/null +++ b/python-package/insightface/utils/face_align.py @@ -0,0 +1,88 @@ + +import cv2 +import numpy as np +from skimage import transform as trans + +src1 = np.array([ + [51.642,50.115], + [57.617,49.990], + [35.740,69.007], + [51.157,89.050], + [57.025,89.702]], dtype=np.float32) +#<--left +src2 = np.array([ + [45.031,50.118], + [65.568,50.872], + [39.677,68.111], + [45.177,86.190], + [64.246,86.758]], dtype=np.float32) + +#---frontal +src3 = np.array([ + [39.730,51.138], + [72.270,51.138], + [56.000,68.493], + [42.463,87.010], + [69.537,87.010]], dtype=np.float32) + +#-->right +src4 = np.array([ + [46.845,50.872], + [67.382,50.118], + [72.737,68.111], + [48.167,86.758], + [67.236,86.190]], dtype=np.float32) + +#-->right profile +src5 = np.array([ + [54.796,49.990], + [60.771,50.115], + [76.673,69.007], + [55.388,89.702], + [61.257,89.050]], dtype=np.float32) + +src = np.array([src1,src2,src3,src4,src5]) +src_map = {112 : src, 224 : src*2} + +arcface_src = np.array([ + [38.2946, 51.6963], + [73.5318, 51.5014], + [56.0252, 71.7366], + [41.5493, 92.3655], + [70.7299, 92.2041] ], dtype=np.float32 ) + +arcface_src = np.expand_dims(arcface_src, axis=0) + +# In[66]: + +# lmk is prediction; src is template +def estimate_norm(lmk, image_size = 112, mode='arcface'): + assert lmk.shape==(5,2) + tform = trans.SimilarityTransform() + lmk_tran = np.insert(lmk, 2, values=np.ones(5), axis=1) + min_M = [] + min_index = [] + min_error = float('inf') + if mode=='arcface': + assert image_size==112 + src = arcface_src + else: + src = src_map[image_size] + for i in np.arange(src.shape[0]): + tform.estimate(lmk, src[i]) + M = tform.params[0:2,:] + results = np.dot(M, lmk_tran.T) + results = results.T + error = np.sum(np.sqrt(np.sum((results - src[i]) ** 2,axis=1))) +# print(error) + if error< min_error: + min_error = error + min_M = M + min_index = i + return min_M, min_index + +def norm_crop(img, landmark, image_size=112, mode='arcface'): + M, pose_index = estimate_norm(landmark, image_size, mode) + warped = cv2.warpAffine(img,M, (image_size, image_size), borderValue = 0.0) + return warped +