From b774d6a1b7c66655f07cc59ce98007ff2c19a11d Mon Sep 17 00:00:00 2001 From: nttstar Date: Fri, 6 Nov 2020 13:59:21 +0800 Subject: [PATCH] refine repo structure --- 3rdparty/operator/amsoftmax-inl.h | 287 ----- 3rdparty/operator/amsoftmax.cc | 64 - 3rdparty/operator/amsoftmax.cu | 195 --- 3rdparty/operator/lsoftmax-inl.h | 377 ------ 3rdparty/operator/lsoftmax.cc | 75 -- 3rdparty/operator/lsoftmax.cu | 322 ----- Evaluation/IJB/IJB_1N.py | 314 ----- Evaluation/Megaface/gen_megaface.py | 179 --- Evaluation/Megaface/remove_noises.py | 156 --- PRNet.mxnet/README.md | 6 - PRNet.mxnet/config.py | 89 -- PRNet.mxnet/data.py | 164 --- PRNet.mxnet/metric.py | 99 -- PRNet.mxnet/symbol/sym_heatmap.py | 435 ------- PRNet.mxnet/train.py | 215 --- README.md | 8 +- RetinaFace/rcnn/PY_OP/cascade_refine.py | 453 ------- RetinaFace/rcnn/PY_OP/rpn_fpn_ohem3.py | 167 --- RetinaFace/rcnn/core/callback.py | 13 - RetinaFace/rcnn/dataset/retinaface.py | 186 --- RetinaFace/rcnn/io/image.py | 808 ------------ RetinaFace/rcnn/io/rpn.py | 765 ----------- RetinaFace/rcnn/symbol/pyramidbox.py | 427 ------ RetinaFace/rcnn/symbol/symbol_common.py | 472 ------- RetinaFace/rcnn/symbol/symbol_mnet.py | 492 ------- RetinaFace/rcnn/symbol/symbol_resnet.py | 423 ------ RetinaFace/rcnn/symbol/symbol_ssh.py | 365 ------ RetinaFace/rcnn/tools/reeval.py | 50 - RetinaFace/rcnn/tools/test_rcnn.py | 109 -- RetinaFace/rcnn/tools/test_rpn.py | 102 -- RetinaFace/rcnn/tools/train_rcnn.py | 172 --- RetinaFace/rcnn/tools/train_rpn.py | 195 --- RetinaFace/retinaface.py | 698 ---------- RetinaFace/test.py | 60 - RetinaFace/test_widerface.py | 201 --- RetinaFace/train.py | 372 ------ RetinaFaceAntiCov/retinaface_cov.py | 637 --------- RetinaFaceAntiCov/test.py | 65 - alignment/coordinateReg/image_infer.py | 210 +-- alignment/heatmapReg/data.py | 496 +++---- alignment/heatmapReg/img_helper.py | 64 +- alignment/heatmapReg/metric.py | 182 +-- alignment/heatmapReg/optimizer.py | 31 +- alignment/heatmapReg/sample_config.py | 12 +- alignment/heatmapReg/symbol/sym_heatmap.py | 1147 ++++++++++++----- alignment/heatmapReg/test.py | 133 +- alignment/heatmapReg/test_rec_nme.py | 48 +- alignment/heatmapReg/train.py | 359 +++--- {IFRT => challenges/IFRT}/README.md | 0 .../iccv19-lfr}/README.md | 0 challenges/iccv19-lfr/gen_image_feature.py | 157 +++ challenges/iccv19-lfr/gen_video_feature.py | 215 +++ common/face_align.py | 88 -- common/flops_counter.py | 114 -- datasets/README.md | 1 - deploy/benchmark.py | 27 +- deploy/convert_onnx.py | 29 +- deploy/face_embedding.py | 140 +- deploy/face_model.py | 161 +-- deploy/ga_merge.py | 51 +- deploy/helper.py | 58 +- deploy/model_slim.py | 19 +- deploy/mtcnn_detector.py | 289 +++-- deploy/test.py | 22 +- {RetinaFace => detection/RetinaFace}/Makefile | 0 .../RetinaFace}/README.md | 0 .../RetinaFace}/rcnn/PY_OP/__init__.py | 0 .../RetinaFace/rcnn/PY_OP/cascade_refine.py | 518 ++++++++ .../RetinaFace/rcnn/PY_OP/rpn_fpn_ohem3.py | 175 +++ .../RetinaFace}/rcnn/__init__.py | 0 .../RetinaFace}/rcnn/core/__init__.py | 0 detection/RetinaFace/rcnn/core/callback.py | 16 + .../RetinaFace}/rcnn/core/loader.py | 295 +++-- .../RetinaFace}/rcnn/core/metric.py | 114 +- .../RetinaFace}/rcnn/core/module.py | 94 +- .../RetinaFace}/rcnn/core/module_bak.py | 95 +- .../RetinaFace}/rcnn/core/tester.py | 222 ++-- .../RetinaFace}/rcnn/cython/.gitignore | 0 .../RetinaFace}/rcnn/cython/__init__.py | 0 .../RetinaFace}/rcnn/cython/anchors.pyx | 0 .../RetinaFace}/rcnn/cython/bbox.pyx | 0 .../RetinaFace}/rcnn/cython/cpu_nms.pyx | 0 .../RetinaFace}/rcnn/cython/gpu_nms.hpp | 0 .../RetinaFace}/rcnn/cython/gpu_nms.pyx | 0 .../RetinaFace}/rcnn/cython/nms_kernel.cu | 0 .../RetinaFace}/rcnn/cython/setup.py | 70 +- .../RetinaFace}/rcnn/dataset/__init__.py | 0 .../RetinaFace}/rcnn/dataset/ds_utils.py | 0 .../RetinaFace}/rcnn/dataset/imdb.py | 157 ++- .../RetinaFace/rcnn/dataset/retinaface.py | 197 +++ .../RetinaFace}/rcnn/io/__init__.py | 0 detection/RetinaFace/rcnn/io/image.py | 886 +++++++++++++ .../RetinaFace}/rcnn/io/rcnn.py | 258 ++-- detection/RetinaFace/rcnn/io/rpn.py | 887 +++++++++++++ .../RetinaFace}/rcnn/logger.py | 0 .../RetinaFace}/rcnn/processing/__init__.py | 0 .../rcnn/processing/assign_levels.py | 11 +- .../rcnn/processing/bbox_regression.py | 28 +- .../rcnn/processing/bbox_transform.py | 73 +- .../rcnn/processing/generate_anchor.py | 57 +- .../RetinaFace}/rcnn/processing/nms.py | 3 + .../RetinaFace}/rcnn/pycocotools/UPSTREAM_REV | 0 .../RetinaFace}/rcnn/pycocotools/__init__.py | 0 .../RetinaFace}/rcnn/pycocotools/_mask.c | 0 .../RetinaFace}/rcnn/pycocotools/_mask.pyx | 0 .../RetinaFace}/rcnn/pycocotools/coco.py | 143 +- .../RetinaFace}/rcnn/pycocotools/cocoeval.py | 332 +++-- .../RetinaFace}/rcnn/pycocotools/mask.py | 10 +- .../RetinaFace}/rcnn/pycocotools/maskApi.c | 0 .../RetinaFace}/rcnn/pycocotools/maskApi.h | 0 .../RetinaFace}/rcnn/pycocotools/setup.py | 4 +- .../RetinaFace}/rcnn/sample_config.py | 141 +- .../RetinaFace}/rcnn/symbol/__init__.py | 0 .../RetinaFace/rcnn/symbol/pyramidbox.py | 489 +++++++ .../RetinaFace/rcnn/symbol/symbol_common.py | 757 +++++++++++ .../rcnn/symbol/symbol_common.py.bak | 0 .../RetinaFace/rcnn/symbol/symbol_mnet.py | 834 ++++++++++++ .../rcnn/symbol/symbol_mnet.py.bak | 0 .../RetinaFace/rcnn/symbol/symbol_resnet.py | 827 ++++++++++++ .../RetinaFace/rcnn/symbol/symbol_ssh.py | 725 +++++++++++ .../RetinaFace}/rcnn/tools/__init__.py | 0 detection/RetinaFace/rcnn/tools/reeval.py | 68 + detection/RetinaFace/rcnn/tools/test_rcnn.py | 161 +++ detection/RetinaFace/rcnn/tools/test_rpn.py | 160 +++ detection/RetinaFace/rcnn/tools/train_rcnn.py | 286 ++++ detection/RetinaFace/rcnn/tools/train_rpn.py | 300 +++++ .../RetinaFace}/rcnn/utils/__init__.py | 0 .../RetinaFace}/rcnn/utils/combine_model.py | 0 .../RetinaFace}/rcnn/utils/load_data.py | 21 +- .../RetinaFace}/rcnn/utils/load_model.py | 0 .../RetinaFace}/rcnn/utils/save_model.py | 4 +- detection/RetinaFace/retinaface.py | 839 ++++++++++++ detection/RetinaFace/test.py | 63 + detection/RetinaFace/test_widerface.py | 259 ++++ detection/RetinaFace/train.py | 502 ++++++++ .../RetinaFaceAntiCov}/README.md | 0 .../rcnn/processing/__init__.py | 0 .../rcnn/processing/assign_levels.py | 11 +- .../rcnn/processing/bbox_regression.py | 28 +- .../rcnn/processing/bbox_transform.py | 73 +- .../rcnn/processing/bbox_transform.py.orig | 0 .../rcnn/processing/generate_anchor.py | 57 +- .../RetinaFaceAntiCov}/rcnn/processing/nms.py | 3 + detection/RetinaFaceAntiCov/retinaface_cov.py | 752 +++++++++++ detection/RetinaFaceAntiCov/test.py | 66 + .../IJB/IJBB_Evaluation_MS1MV2.ipynb | 0 .../IJB/IJBB_Evaluation_VGG2.ipynb | 0 .../IJB/IJBC_Evaluation_MS1MV2.ipynb | 0 .../IJB/IJBC_Evaluation_VGG2.ipynb | 0 {Evaluation => evaluation}/IJB/IJB_11.py | 205 +-- evaluation/IJB/IJB_1N.py | 366 ++++++ {Evaluation => evaluation}/IJB/README.md | 0 {Evaluation => evaluation}/IJB/example.sh | 0 {Evaluation => evaluation}/IJB/readme.txt | 0 {Evaluation => evaluation}/Megaface/README.md | 0 evaluation/Megaface/gen_megaface.py | 196 +++ evaluation/Megaface/remove_noises.py | 182 +++ {Evaluation => evaluation}/Megaface/run.sh | 0 gender-age/data.py | 243 ++-- gender-age/face_model.py | 144 ++- gender-age/fmobilenet.py | 270 +++- gender-age/fresnet.py | 1044 +++++++++++---- gender-age/helper.py | 58 +- gender-age/mtcnn_detector.py | 289 +++-- gender-age/symbol_utils.py | 440 +++++-- gender-age/test.py | 19 +- gender-age/train.py | 519 +++++--- iccv19-challenge/gen_image_feature.py | 153 --- iccv19-challenge/gen_video_feature.py | 204 --- models/README.md | 1 - python-package/insightface/__init__.py | 4 +- .../insightface/app/face_analysis.py | 57 +- .../insightface/model_zoo/face_detection.py | 204 +-- .../insightface/model_zoo/face_genderage.py | 38 +- .../insightface/model_zoo/face_recognition.py | 37 +- .../insightface/model_zoo/model_store.py | 37 +- .../insightface/model_zoo/model_zoo.py | 1 - python-package/insightface/utils/download.py | 17 +- .../insightface/utils/face_align.py | 119 +- .../insightface/utils/filesystem.py | 18 +- python-package/setup.py | 11 +- recognition/ArcFace/image_iter.py | 315 ++--- recognition/ArcFace/metric.py | 82 +- recognition/ArcFace/parall_module_local_v1.py | 411 +++--- recognition/ArcFace/sample_config.py | 34 +- recognition/ArcFace/train.py | 637 +++++---- recognition/ArcFace/train_parall.py | 505 +++++--- recognition/ArcFace/triplet_image_iter.py | 845 ++++++------ recognition/ArcFace/verification.py | 1019 ++++++++------- recognition/SubCenter-ArcFace/drop.py | 360 +++--- recognition/SubCenter-ArcFace/image_iter.py | 300 ++--- .../parall_module_local_v1.py | 479 ++++--- .../SubCenter-ArcFace/sample_config.py | 33 +- recognition/SubCenter-ArcFace/train_parall.py | 492 ++++--- recognition/common/build_eval_pack.py | 156 +-- recognition/common/face_align.py | 71 + recognition/common/flops_counter.py | 168 +-- recognition/common/rec2image.py | 85 +- recognition/common/rec_builder.py | 33 +- recognition/common/verification.py | 555 ++++---- recognition/partial_fc/mxnet/callbacks.py | 58 +- recognition/partial_fc/mxnet/default.py | 5 +- .../partial_fc/mxnet/evaluation/align_ijb.py | 12 +- .../partial_fc/mxnet/evaluation/ijb.py | 147 ++- .../partial_fc/mxnet/evaluation/lfw.py | 350 ++--- .../mxnet/evaluation/verification.py | 186 ++- recognition/partial_fc/mxnet/image_iter.py | 59 +- recognition/partial_fc/mxnet/memory_bank.py | 28 +- recognition/partial_fc/mxnet/memory_module.py | 237 ++-- .../partial_fc/mxnet/memory_samplers.py | 15 +- .../partial_fc/mxnet/memory_scheduler.py | 2 +- .../partial_fc/mxnet/memory_softmax.py | 7 +- recognition/partial_fc/mxnet/optimizer.py | 14 +- .../partial_fc/mxnet/symbol/memonger.py | 17 +- recognition/partial_fc/mxnet/symbol/resnet.py | 923 ++++++++++--- .../partial_fc/mxnet/symbol/symbol_utils.py | 514 ++++++-- recognition/partial_fc/mxnet/train_memory.py | 46 +- recognition/partial_fc/pytorch/IJB/IJB_11.py | 105 +- .../partial_fc/pytorch/IJB/IJB_11_Batch.py | 115 +- .../partial_fc/pytorch/IJB/IJB_11_Figure.py | 54 +- .../partial_fc/pytorch/IJB/IJB_11_test.py | 99 +- recognition/partial_fc/pytorch/IJB/IJB_1N.py | 354 ++--- recognition/partial_fc/pytorch/IJB/plot_ax.py | 72 +- .../pytorch/IJB/recognition/__init__.py | 1 - .../pytorch/IJB/recognition/embedding_test.py | 31 +- .../partial_fc/pytorch/backbones/__init__.py | 2 +- .../partial_fc/pytorch/backbones/iresnet.py | 128 +- recognition/partial_fc/pytorch/config.py | 4 +- recognition/partial_fc/pytorch/dataset.py | 8 +- .../partial_fc/pytorch/partial_classifier.py | 22 +- recognition/partial_fc/pytorch/partial_fc.py | 73 +- recognition/partial_fc/pytorch/sgd.py | 28 +- recognition/partial_fc/unpack_glint360k.py | 3 +- recognition/symbol/fdensenet.py | 62 +- recognition/symbol/fmnasnet.py | 150 ++- recognition/symbol/fmobilefacenet.py | 236 +++- recognition/symbol/fmobilenet.py | 276 +++- recognition/symbol/fresnet.py | 1087 ++++++++++++---- recognition/symbol/memonger.py | 17 +- recognition/symbol/memonger_v2.py | 55 +- recognition/symbol/symbol_utils.py | 645 ++++++--- recognition/symbol/vargfacenet.py | 28 +- .../tools/cpp-align}/FacePreprocess.h | 0 src/age_iter.py | 269 ---- src/align/__init__.py | 0 src/align/align_celeb.py | 225 ---- src/align/align_dataset.py | 137 -- src/align/align_dataset_mtcnn.py | 143 -- src/align/align_dlib.py | 204 --- src/align/align_facescrub.py | 289 ----- src/align/align_insight.py | 251 ---- src/align/align_lfw.py | 161 --- src/align/align_megaface.py | 240 ---- src/align/det1.npy | Bin 27368 -> 0 bytes src/align/det2.npy | Bin 401681 -> 0 bytes src/align/det3.npy | Bin 1557360 -> 0 bytes src/align/detect_face.py | 848 ------------ src/api/app.py | 99 -- src/api/face_model.py | 203 --- src/common/__init__.py | 0 src/common/face_image.py | 271 ---- src/common/face_preprocess.py | 113 -- src/common/noise_sgd.py | 38 - src/data.py | 1016 --------------- src/data/age_merge.py | 125 -- src/data/agedb2pack.py | 158 --- src/data/agedb2pack2.py | 143 -- src/data/cfp2pack.py | 145 --- src/data/dataset_c2c.py | 136 -- src/data/dataset_clean.py | 188 --- src/data/dataset_info.py | 40 - src/data/dataset_merge.py | 293 ----- src/data/dataset_relabel.py | 70 - src/data/dir2lst.py | 14 - src/data/dir2lst_ytf.py | 32 - src/data/dir2rec.py | 205 --- src/data/face2rec2.py | 267 ---- src/data/glint2lst.py | 42 - src/data/lfw2pack.py | 35 - src/eval/do_ver.sh | 3 - src/eval/gen_glint.py | 155 --- src/eval/lfw.py | 279 ---- src/eval/verification.py | 590 --------- src/eval/ytf.py | 302 ----- src/eval/ytf_badcases.py | 75 -- src/image_iter.py | 316 ----- src/losses/center_loss.py | 123 -- src/megaface/README.md | 8 - src/megaface/facescrub_noises.txt | 606 --------- src/megaface/gen_megaface.py | 236 ---- src/megaface/megaface_noises.txt | 720 ----------- src/megaface/remove_noises.py | 175 --- src/symbols/fdensenet.py | 192 --- src/symbols/fdpn.py | 229 ---- src/symbols/finception_resnet_v2.py | 168 --- src/symbols/fmobilefacenet.py | 67 - src/symbols/fmobilenet.py | 84 -- src/symbols/fmobilenetv2.py | 106 -- src/symbols/fnasnet.py | 575 --------- src/symbols/fresnet.py | 601 --------- src/symbols/fxception.py | 151 --- src/symbols/spherenet.py | 62 - src/symbols/symbol_utils.py | 182 --- src/train.py | 1026 --------------- src/train.sh | 15 - src/train_softmax.py | 598 --------- src/train_triplet.py | 402 ------ src/triplet_image_iter.py | 608 --------- src/utils/benchmark.py | 101 -- 309 files changed, 24974 insertions(+), 34253 deletions(-) delete mode 100644 3rdparty/operator/amsoftmax-inl.h delete mode 100644 3rdparty/operator/amsoftmax.cc delete mode 100644 3rdparty/operator/amsoftmax.cu delete mode 100644 3rdparty/operator/lsoftmax-inl.h delete mode 100644 3rdparty/operator/lsoftmax.cc delete mode 100644 3rdparty/operator/lsoftmax.cu delete mode 100644 Evaluation/IJB/IJB_1N.py delete mode 100644 Evaluation/Megaface/gen_megaface.py delete mode 100644 Evaluation/Megaface/remove_noises.py delete mode 100644 PRNet.mxnet/README.md delete mode 100644 PRNet.mxnet/config.py delete mode 100644 PRNet.mxnet/data.py delete mode 100644 PRNet.mxnet/metric.py delete mode 100644 PRNet.mxnet/symbol/sym_heatmap.py delete mode 100644 PRNet.mxnet/train.py delete mode 100644 RetinaFace/rcnn/PY_OP/cascade_refine.py delete mode 100644 RetinaFace/rcnn/PY_OP/rpn_fpn_ohem3.py delete mode 100644 RetinaFace/rcnn/core/callback.py delete mode 100644 RetinaFace/rcnn/dataset/retinaface.py delete mode 100644 RetinaFace/rcnn/io/image.py delete mode 100644 RetinaFace/rcnn/io/rpn.py delete mode 100644 RetinaFace/rcnn/symbol/pyramidbox.py delete mode 100644 RetinaFace/rcnn/symbol/symbol_common.py delete mode 100644 RetinaFace/rcnn/symbol/symbol_mnet.py delete mode 100644 RetinaFace/rcnn/symbol/symbol_resnet.py delete mode 100644 RetinaFace/rcnn/symbol/symbol_ssh.py delete mode 100644 RetinaFace/rcnn/tools/reeval.py delete mode 100644 RetinaFace/rcnn/tools/test_rcnn.py delete mode 100644 RetinaFace/rcnn/tools/test_rpn.py delete mode 100644 RetinaFace/rcnn/tools/train_rcnn.py delete mode 100644 RetinaFace/rcnn/tools/train_rpn.py delete mode 100644 RetinaFace/retinaface.py delete mode 100644 RetinaFace/test.py delete mode 100644 RetinaFace/test_widerface.py delete mode 100644 RetinaFace/train.py delete mode 100644 RetinaFaceAntiCov/retinaface_cov.py delete mode 100644 RetinaFaceAntiCov/test.py rename {IFRT => challenges/IFRT}/README.md (100%) rename {iccv19-challenge => challenges/iccv19-lfr}/README.md (100%) create mode 100644 challenges/iccv19-lfr/gen_image_feature.py create mode 100644 challenges/iccv19-lfr/gen_video_feature.py delete mode 100644 common/face_align.py delete mode 100644 common/flops_counter.py delete mode 100644 datasets/README.md rename {RetinaFace => detection/RetinaFace}/Makefile (100%) rename {RetinaFace => detection/RetinaFace}/README.md (100%) rename {RetinaFace => detection/RetinaFace}/rcnn/PY_OP/__init__.py (100%) create mode 100644 detection/RetinaFace/rcnn/PY_OP/cascade_refine.py create mode 100644 detection/RetinaFace/rcnn/PY_OP/rpn_fpn_ohem3.py rename {RetinaFace => detection/RetinaFace}/rcnn/__init__.py (100%) rename {RetinaFace => detection/RetinaFace}/rcnn/core/__init__.py (100%) create mode 100644 detection/RetinaFace/rcnn/core/callback.py rename {RetinaFace => detection/RetinaFace}/rcnn/core/loader.py (61%) rename {RetinaFace => detection/RetinaFace}/rcnn/core/metric.py (54%) rename {RetinaFace => detection/RetinaFace}/rcnn/core/module.py (69%) rename {RetinaFace => detection/RetinaFace}/rcnn/core/module_bak.py (69%) rename {RetinaFace => detection/RetinaFace}/rcnn/core/tester.py (73%) rename {RetinaFace => detection/RetinaFace}/rcnn/cython/.gitignore (100%) rename {RetinaFace => detection/RetinaFace}/rcnn/cython/__init__.py (100%) rename {RetinaFace => detection/RetinaFace}/rcnn/cython/anchors.pyx (100%) rename {RetinaFace => detection/RetinaFace}/rcnn/cython/bbox.pyx (100%) rename {RetinaFace => detection/RetinaFace}/rcnn/cython/cpu_nms.pyx (100%) rename {RetinaFace => detection/RetinaFace}/rcnn/cython/gpu_nms.hpp (100%) rename {RetinaFace => detection/RetinaFace}/rcnn/cython/gpu_nms.pyx (100%) rename {RetinaFace => detection/RetinaFace}/rcnn/cython/nms_kernel.cu (100%) rename {RetinaFace => detection/RetinaFace}/rcnn/cython/setup.py (75%) rename {RetinaFace => detection/RetinaFace}/rcnn/dataset/__init__.py (100%) rename {RetinaFace => detection/RetinaFace}/rcnn/dataset/ds_utils.py (100%) rename {RetinaFace => detection/RetinaFace}/rcnn/dataset/imdb.py (70%) create mode 100644 detection/RetinaFace/rcnn/dataset/retinaface.py rename {RetinaFace => detection/RetinaFace}/rcnn/io/__init__.py (100%) create mode 100644 detection/RetinaFace/rcnn/io/image.py rename {RetinaFace => detection/RetinaFace}/rcnn/io/rcnn.py (74%) create mode 100644 detection/RetinaFace/rcnn/io/rpn.py rename {RetinaFace => detection/RetinaFace}/rcnn/logger.py (100%) rename {RetinaFace => detection/RetinaFace}/rcnn/processing/__init__.py (100%) rename {RetinaFaceAntiCov => detection/RetinaFace}/rcnn/processing/assign_levels.py (73%) rename {RetinaFaceAntiCov => detection/RetinaFace}/rcnn/processing/bbox_regression.py (93%) rename {RetinaFace => detection/RetinaFace}/rcnn/processing/bbox_transform.py (79%) rename {RetinaFaceAntiCov => detection/RetinaFace}/rcnn/processing/generate_anchor.py (70%) rename {RetinaFace => detection/RetinaFace}/rcnn/processing/nms.py (99%) rename {RetinaFace => detection/RetinaFace}/rcnn/pycocotools/UPSTREAM_REV (100%) rename {RetinaFace => detection/RetinaFace}/rcnn/pycocotools/__init__.py (100%) rename {RetinaFace => detection/RetinaFace}/rcnn/pycocotools/_mask.c (100%) rename {RetinaFace => detection/RetinaFace}/rcnn/pycocotools/_mask.pyx (100%) rename {RetinaFace => detection/RetinaFace}/rcnn/pycocotools/coco.py (77%) rename {RetinaFace => detection/RetinaFace}/rcnn/pycocotools/cocoeval.py (64%) rename {RetinaFace => detection/RetinaFace}/rcnn/pycocotools/mask.py (97%) rename {RetinaFace => detection/RetinaFace}/rcnn/pycocotools/maskApi.c (100%) rename {RetinaFace => detection/RetinaFace}/rcnn/pycocotools/maskApi.h (100%) rename {RetinaFace => detection/RetinaFace}/rcnn/pycocotools/setup.py (88%) rename {RetinaFace => detection/RetinaFace}/rcnn/sample_config.py (74%) rename {RetinaFace => detection/RetinaFace}/rcnn/symbol/__init__.py (100%) create mode 100644 detection/RetinaFace/rcnn/symbol/pyramidbox.py create mode 100644 detection/RetinaFace/rcnn/symbol/symbol_common.py rename {RetinaFace => detection/RetinaFace}/rcnn/symbol/symbol_common.py.bak (100%) create mode 100644 detection/RetinaFace/rcnn/symbol/symbol_mnet.py rename {RetinaFace => detection/RetinaFace}/rcnn/symbol/symbol_mnet.py.bak (100%) create mode 100644 detection/RetinaFace/rcnn/symbol/symbol_resnet.py create mode 100644 detection/RetinaFace/rcnn/symbol/symbol_ssh.py rename {RetinaFace => detection/RetinaFace}/rcnn/tools/__init__.py (100%) create mode 100644 detection/RetinaFace/rcnn/tools/reeval.py create mode 100644 detection/RetinaFace/rcnn/tools/test_rcnn.py create mode 100644 detection/RetinaFace/rcnn/tools/test_rpn.py create mode 100644 detection/RetinaFace/rcnn/tools/train_rcnn.py create mode 100644 detection/RetinaFace/rcnn/tools/train_rpn.py rename {RetinaFace => detection/RetinaFace}/rcnn/utils/__init__.py (100%) rename {RetinaFace => detection/RetinaFace}/rcnn/utils/combine_model.py (100%) rename {RetinaFace => detection/RetinaFace}/rcnn/utils/load_data.py (74%) rename {RetinaFace => detection/RetinaFace}/rcnn/utils/load_model.py (100%) rename {RetinaFace => detection/RetinaFace}/rcnn/utils/save_model.py (81%) create mode 100644 detection/RetinaFace/retinaface.py create mode 100644 detection/RetinaFace/test.py create mode 100644 detection/RetinaFace/test_widerface.py create mode 100644 detection/RetinaFace/train.py rename {RetinaFaceAntiCov => detection/RetinaFaceAntiCov}/README.md (100%) rename {RetinaFaceAntiCov => detection/RetinaFaceAntiCov}/rcnn/processing/__init__.py (100%) rename {RetinaFace => detection/RetinaFaceAntiCov}/rcnn/processing/assign_levels.py (73%) rename {RetinaFace => detection/RetinaFaceAntiCov}/rcnn/processing/bbox_regression.py (93%) rename {RetinaFaceAntiCov => detection/RetinaFaceAntiCov}/rcnn/processing/bbox_transform.py (79%) rename {RetinaFaceAntiCov => detection/RetinaFaceAntiCov}/rcnn/processing/bbox_transform.py.orig (100%) rename {RetinaFace => detection/RetinaFaceAntiCov}/rcnn/processing/generate_anchor.py (70%) rename {RetinaFaceAntiCov => detection/RetinaFaceAntiCov}/rcnn/processing/nms.py (99%) create mode 100644 detection/RetinaFaceAntiCov/retinaface_cov.py create mode 100644 detection/RetinaFaceAntiCov/test.py rename {Evaluation => evaluation}/IJB/IJBB_Evaluation_MS1MV2.ipynb (100%) rename {Evaluation => evaluation}/IJB/IJBB_Evaluation_VGG2.ipynb (100%) rename {Evaluation => evaluation}/IJB/IJBC_Evaluation_MS1MV2.ipynb (100%) rename {Evaluation => evaluation}/IJB/IJBC_Evaluation_VGG2.ipynb (100%) rename {Evaluation => evaluation}/IJB/IJB_11.py (62%) create mode 100644 evaluation/IJB/IJB_1N.py rename {Evaluation => evaluation}/IJB/README.md (100%) rename {Evaluation => evaluation}/IJB/example.sh (100%) rename {Evaluation => evaluation}/IJB/readme.txt (100%) rename {Evaluation => evaluation}/Megaface/README.md (100%) create mode 100644 evaluation/Megaface/gen_megaface.py create mode 100644 evaluation/Megaface/remove_noises.py rename {Evaluation => evaluation}/Megaface/run.sh (100%) delete mode 100644 iccv19-challenge/gen_image_feature.py delete mode 100644 iccv19-challenge/gen_video_feature.py delete mode 100644 models/README.md create mode 100644 recognition/common/face_align.py rename {cpp-align => recognition/tools/cpp-align}/FacePreprocess.h (100%) delete mode 100644 src/age_iter.py delete mode 100644 src/align/__init__.py delete mode 100644 src/align/align_celeb.py delete mode 100644 src/align/align_dataset.py delete mode 100644 src/align/align_dataset_mtcnn.py delete mode 100644 src/align/align_dlib.py delete mode 100644 src/align/align_facescrub.py delete mode 100644 src/align/align_insight.py delete mode 100644 src/align/align_lfw.py delete mode 100644 src/align/align_megaface.py delete mode 100644 src/align/det1.npy delete mode 100644 src/align/det2.npy delete mode 100644 src/align/det3.npy delete mode 100644 src/align/detect_face.py delete mode 100644 src/api/app.py delete mode 100644 src/api/face_model.py delete mode 100644 src/common/__init__.py delete mode 100644 src/common/face_image.py delete mode 100644 src/common/face_preprocess.py delete mode 100644 src/common/noise_sgd.py delete mode 100644 src/data.py delete mode 100644 src/data/age_merge.py delete mode 100644 src/data/agedb2pack.py delete mode 100644 src/data/agedb2pack2.py delete mode 100644 src/data/cfp2pack.py delete mode 100644 src/data/dataset_c2c.py delete mode 100644 src/data/dataset_clean.py delete mode 100644 src/data/dataset_info.py delete mode 100644 src/data/dataset_merge.py delete mode 100644 src/data/dataset_relabel.py delete mode 100644 src/data/dir2lst.py delete mode 100644 src/data/dir2lst_ytf.py delete mode 100644 src/data/dir2rec.py delete mode 100644 src/data/face2rec2.py delete mode 100644 src/data/glint2lst.py delete mode 100644 src/data/lfw2pack.py delete mode 100755 src/eval/do_ver.sh delete mode 100644 src/eval/gen_glint.py delete mode 100644 src/eval/lfw.py delete mode 100644 src/eval/verification.py delete mode 100644 src/eval/ytf.py delete mode 100644 src/eval/ytf_badcases.py delete mode 100644 src/image_iter.py delete mode 100644 src/losses/center_loss.py delete mode 100644 src/megaface/README.md delete mode 100644 src/megaface/facescrub_noises.txt delete mode 100644 src/megaface/gen_megaface.py delete mode 100644 src/megaface/megaface_noises.txt delete mode 100644 src/megaface/remove_noises.py delete mode 100644 src/symbols/fdensenet.py delete mode 100644 src/symbols/fdpn.py delete mode 100644 src/symbols/finception_resnet_v2.py delete mode 100644 src/symbols/fmobilefacenet.py delete mode 100644 src/symbols/fmobilenet.py delete mode 100644 src/symbols/fmobilenetv2.py delete mode 100644 src/symbols/fnasnet.py delete mode 100644 src/symbols/fresnet.py delete mode 100644 src/symbols/fxception.py delete mode 100644 src/symbols/spherenet.py delete mode 100644 src/symbols/symbol_utils.py delete mode 100644 src/train.py delete mode 100755 src/train.sh delete mode 100644 src/train_softmax.py delete mode 100644 src/train_triplet.py delete mode 100644 src/triplet_image_iter.py delete mode 100644 src/utils/benchmark.py diff --git a/3rdparty/operator/amsoftmax-inl.h b/3rdparty/operator/amsoftmax-inl.h deleted file mode 100644 index d899bbb..0000000 --- a/3rdparty/operator/amsoftmax-inl.h +++ /dev/null @@ -1,287 +0,0 @@ -/*! - * Copyright (c) 2018 by Contributors - * \file amsoftmax-inl.h - * \brief AmSoftmax from - * \author Jia Guo - */ -#ifndef MXNET_OPERATOR_AMSOFTMAX_INL_H_ -#define MXNET_OPERATOR_AMSOFTMAX_INL_H_ - -#include -#include -#include -#include -#include -#include -#include -#include "./operator_common.h" - -namespace mxnet { -namespace op { - -namespace amsoftmax_enum { -enum AmSoftmaxOpInputs {kData, kWeight, kLabel}; -enum AmSoftmaxOpOutputs {kOut, kOOut}; -enum AmSoftmaxResource {kTempSpace}; -} - -struct AmSoftmaxParam : public dmlc::Parameter { - float margin; - float s; - int num_hidden; - int verbose; - float eps; - DMLC_DECLARE_PARAMETER(AmSoftmaxParam) { - DMLC_DECLARE_FIELD(margin).set_default(0.5).set_lower_bound(0.0) - .describe("AmSoftmax margin"); - DMLC_DECLARE_FIELD(s).set_default(64.0).set_lower_bound(1.0) - .describe("s to X"); - DMLC_DECLARE_FIELD(num_hidden).set_lower_bound(1) - .describe("Number of hidden nodes of the output"); - DMLC_DECLARE_FIELD(verbose).set_default(0) - .describe("Log for beta change"); - DMLC_DECLARE_FIELD(eps).set_default(1e-10f) - .describe("l2 eps"); - } -}; - -template -class AmSoftmaxOp : public Operator { - public: - explicit AmSoftmaxOp(AmSoftmaxParam param) { - this->param_ = param; - count_ = 0; - } - - virtual void Forward(const OpContext &ctx, - const std::vector &in_data, - const std::vector &req, - const std::vector &out_data, - const std::vector &aux_args) { - using namespace mshadow; - using namespace mshadow::expr; - CHECK_EQ(in_data.size(), 3); - CHECK_EQ(out_data.size(), 2); - CHECK_EQ(req.size(), 2); - CHECK_EQ(req[amsoftmax_enum::kOut], kWriteTo); - Stream *stream = ctx.get_stream(); - const int n = in_data[amsoftmax_enum::kData].size(0); //batch size - const int m = in_data[amsoftmax_enum::kWeight].size(0);//num classes - Tensor x = in_data[amsoftmax_enum::kData].FlatTo2D(stream); - Tensor w = in_data[amsoftmax_enum::kWeight].FlatTo2D(stream); - Tensor label = in_data[amsoftmax_enum::kLabel].get_with_shape(Shape1(n), stream); - Tensor out = out_data[amsoftmax_enum::kOut].FlatTo2D(stream); - Tensor oout = out_data[amsoftmax_enum::kOOut].get_with_shape(Shape2(n,1), stream); - //Tensor workspace = ctx.requested[amsoftmax_enum::kTempSpace].get_space_typed(Shape2(n, 1), stream); -#if defined(__CUDACC__) - CHECK_EQ(stream->blas_handle_ownership_, Stream::OwnHandle) - << "Must init CuBLAS handle in stream"; -#endif - // original fully connected - out = dot(x, w.T()); - if (ctx.is_train) { - const DType margin = static_cast(param_.margin); - const DType s = static_cast(param_.s); - AmSoftmaxForward(x, w, label, out, oout, margin, s); - } - } - - //virtual void GradNorm(mshadow::Tensor grad, mshadow::Stream* s) { - // using namespace mshadow; - // using namespace mshadow::expr; - // Tensor grad_cpu(grad.shape_); - // AllocSpace(&grad_cpu); - // Copy(grad_cpu, grad, s); - // DType grad_norm = param_.eps; - // for(uint32_t i=0;i grad, mshadow::Stream* s) { - using namespace mshadow; - using namespace mshadow::expr; - Tensor grad_cpu(grad.shape_); - AllocSpace(&grad_cpu); - Copy(grad_cpu, grad, s); - DType grad_norm = param_.eps; - for(uint32_t i=0;i tensor, mshadow::Stream* s) { - using namespace mshadow; - using namespace mshadow::expr; - Tensor tensor_cpu(tensor.shape_); - AllocSpace(&tensor_cpu); - Copy(tensor_cpu, tensor, s); - for(uint32_t i=0;i &out_grad, - const std::vector &in_data, - const std::vector &out_data, - const std::vector &req, - const std::vector &in_grad, - const std::vector &aux_args) { - using namespace mshadow; - using namespace mshadow::expr; - CHECK_EQ(out_grad.size(), 1); - CHECK_EQ(in_data.size(), 3); - CHECK_EQ(out_data.size(), 2); - CHECK_GE(in_grad.size(), 2); - CHECK_GE(req.size(), 2); - CHECK_EQ(req[amsoftmax_enum::kData], kWriteTo); - CHECK_EQ(req[amsoftmax_enum::kWeight], kWriteTo); - Stream *stream = ctx.get_stream(); - const int n = in_data[amsoftmax_enum::kData].size(0); - const int m = in_data[amsoftmax_enum::kWeight].size(0); - Tensor x = in_data[amsoftmax_enum::kData].FlatTo2D(stream); - Tensor w = in_data[amsoftmax_enum::kWeight].FlatTo2D(stream); - Tensor label = in_data[amsoftmax_enum::kLabel].get_with_shape(Shape1(n), stream); - Tensor out = out_data[amsoftmax_enum::kOut].FlatTo2D(stream); - Tensor oout = out_data[amsoftmax_enum::kOOut].get_with_shape(Shape2(n,1), stream); - Tensor o_grad = out_grad[amsoftmax_enum::kOut].FlatTo2D(stream); - Tensor x_grad = in_grad[amsoftmax_enum::kData].FlatTo2D(stream); - Tensor w_grad = in_grad[amsoftmax_enum::kWeight].FlatTo2D(stream); - Tensor workspace = ctx.requested[amsoftmax_enum::kTempSpace].get_space_typed(Shape2(n, 1), stream); -#if defined(__CUDACC__) - CHECK_EQ(stream->blas_handle_ownership_, Stream::OwnHandle) - << "Must init CuBLAS handle in stream"; -#endif - // original fully connected - x_grad = dot(o_grad, w); - w_grad = dot(o_grad.T(), x); - // large margin fully connected - const DType margin = static_cast(param_.margin); - const DType s = static_cast(param_.s); - AmSoftmaxBackward(x, w, label, out, oout, o_grad, x_grad, w_grad, workspace, margin, s); - count_+=1; - if (param_.verbose) { - if(count_%param_.verbose==0) { - DType n = GradNorm(x_grad, stream); - LOG(INFO)<<"x_grad norm:"< -Operator *CreateOp(AmSoftmaxParam param, int dtype); - -#if DMLC_USE_CXX11 -class AmSoftmaxProp : public OperatorProperty { - public: - void Init(const std::vector > &kwargs) override { - param_.Init(kwargs); - } - - std::map GetParams() const override { - return param_.__DICT__(); - } - - std::vector ListArguments() const override { - return {"data", "weight", "label"}; - } - - std::vector ListOutputs() const override { - return {"output", "ooutput"}; - } - - int NumOutputs() const override { - return 2; - } - - int NumVisibleOutputs() const override { - return 1; - } - - bool InferShape(std::vector *in_shape, - std::vector *out_shape, - std::vector *aux_shape) const override { - using namespace mshadow; - CHECK_EQ(in_shape->size(), 3) << "Input:[data, label, weight]"; - const TShape &dshape = in_shape->at(amsoftmax_enum::kData); - const TShape &lshape = in_shape->at(amsoftmax_enum::kLabel); - CHECK_EQ(dshape.ndim(), 2) << "data shape should be (batch_size, feature_dim)"; - CHECK_EQ(lshape.ndim(), 1) << "label shape should be (batch_size,)"; - const int n = dshape[0]; - const int feature_dim = dshape[1]; - const int m = param_.num_hidden; - SHAPE_ASSIGN_CHECK(*in_shape, amsoftmax_enum::kWeight, Shape2(m, feature_dim)); - out_shape->clear(); - out_shape->push_back(Shape2(n, m)); // output - out_shape->push_back(Shape2(n, 1)); // output - aux_shape->clear(); - return true; - } - - std::vector BackwardResource( - const std::vector &in_shape) const override { - return {ResourceRequest::kTempSpace}; - } - - std::vector DeclareBackwardDependency( - const std::vector &out_grad, - const std::vector &in_data, - const std::vector &out_data) const override { - return {out_grad[amsoftmax_enum::kOut], - in_data[amsoftmax_enum::kData], - in_data[amsoftmax_enum::kWeight], in_data[amsoftmax_enum::kLabel]}; - } - - std::string TypeString() const override { - return "AmSoftmax"; - } - - OperatorProperty *Copy() const override { - auto ptr = new AmSoftmaxProp(); - ptr->param_ = param_; - return ptr; - } - - Operator *CreateOperator(Context ctx) const override { - LOG(FATAL) << "Not Implemented."; - return NULL; - } - - Operator *CreateOperatorEx(Context ctx, std::vector *in_shape, - std::vector *in_type) const override; - - private: - AmSoftmaxParam param_; -}; -#endif // DMLC_USE_CXX11 - -} // namespace op -} // namespace mxnet - -#endif diff --git a/3rdparty/operator/amsoftmax.cc b/3rdparty/operator/amsoftmax.cc deleted file mode 100644 index 4075fd3..0000000 --- a/3rdparty/operator/amsoftmax.cc +++ /dev/null @@ -1,64 +0,0 @@ -#include "./amsoftmax-inl.h" - -namespace mshadow { - -template -inline void AmSoftmaxForward(const Tensor &x, - const Tensor &w, - const Tensor &label, - const Tensor &out, - const Tensor &oout, - const DType margin, - const DType s) { - LOG(FATAL) << "Not Implemented."; -} - -template -inline void AmSoftmaxBackward(const Tensor &x, - const Tensor &w, - const Tensor &label, - const Tensor &out, - const Tensor &oout, - const Tensor &o_grad, - const Tensor &x_grad, - const Tensor &w_grad, - const Tensor &workspace, - const DType margin, - const DType s) { - LOG(FATAL) << "Not Implemented."; -} - -} // namespace mshadow - -namespace mxnet { -namespace op { - -template<> -Operator *CreateOp(AmSoftmaxParam param, int dtype) { - Operator *op = NULL; - MSHADOW_REAL_TYPE_SWITCH(dtype, DType, { - op = new AmSoftmaxOp(param); - }) - return op; -} - -Operator *AmSoftmaxProp::CreateOperatorEx(Context ctx, std::vector *in_shape, - std::vector *in_type) const { - std::vector out_shape, aux_shape; - std::vector out_type, aux_type; - CHECK(InferType(in_type, &out_type, &aux_type)); - CHECK(InferShape(in_shape, &out_shape, &aux_shape)); - DO_BIND_DISPATCH(CreateOp, param_, in_type->at(0)); -} - -DMLC_REGISTER_PARAMETER(AmSoftmaxParam); - -MXNET_REGISTER_OP_PROPERTY(AmSoftmax, AmSoftmaxProp) -.describe("AmSoftmax from ") -.add_argument("data", "Symbol", "data") -.add_argument("weight", "Symbol", "weight") -.add_argument("label", "Symbol", "label") -.add_arguments(AmSoftmaxParam::__FIELDS__()); - -} // namespace op -} // namespace mxnet diff --git a/3rdparty/operator/amsoftmax.cu b/3rdparty/operator/amsoftmax.cu deleted file mode 100644 index dfd7f1f..0000000 --- a/3rdparty/operator/amsoftmax.cu +++ /dev/null @@ -1,195 +0,0 @@ -#include "./amsoftmax-inl.h" -#include - -namespace mshadow { -namespace cuda { - -#define CUDA_KERNEL_LOOP(i, n) \ - for (int i = blockIdx.x * blockDim.x + threadIdx.x; \ - i < (n); \ - i += blockDim.x * gridDim.x) - - -template -__global__ void AmSoftmaxForwardKernel(const Tensor x, - const Tensor w, - const Tensor label, - Tensor out, - Tensor oout, - const DType margin, - const DType s) { - const int n = x.size(0); //batch size - const int feature_dim = x.size(1); //embedding size, 512 for example - const int m = w.size(0);//num classes - const DType cos_m = cos(margin); - const DType sin_m = sin(margin); - CUDA_KERNEL_LOOP(i, n) { - const int yi = static_cast(label[i]); - const DType fo_i_yi = out[i][yi]; - oout[i][0] = fo_i_yi; - if(fo_i_yi>=0.0) { - const DType cos_t = fo_i_yi / s; - const DType sin_t = sqrt(1.0-cos_t*cos_t); - out[i][yi] = fo_i_yi*cos_m - (s*sin_t*sin_m); - } - } -} - -template -inline void AmSoftmaxForward(const Tensor &x, - const Tensor &w, - const Tensor &label, - const Tensor &out, - const Tensor &oout, - const DType margin, - const DType s) { - const int n = x.size(0); - const int m = w.size(0); - dim3 dimBlock(kBaseThreadNum); - dim3 dimGrid((n + kBaseThreadNum - 1) / kBaseThreadNum); - AmSoftmaxForwardKernel<<>>(x, w, label, out, oout, margin, s); -} - - -template -__global__ void AmSoftmaxBackwardXKernel(const Tensor x, - const Tensor w, - const Tensor label, - const Tensor out, - const Tensor oout, - const Tensor o_grad, - Tensor x_grad, - const Tensor workspace, - const DType margin, - const DType s) { - const int nthreads = x.size(0) * x.size(1); - //const int nthreads = x.size(0); - const int feature_dim = x.size(1); - const DType cos_m = cos(margin); - const DType nsin_m = sin(margin)*-1.0; - const DType ss = s*s; - CUDA_KERNEL_LOOP(idx, nthreads) { - const int i = idx / feature_dim; - const int l = idx % feature_dim; - //const int i = idx; - const int yi = static_cast(label[i]); - if(oout[i][0]>=0.0) { - //x_grad[i][l] -= o_grad[i][yi] * w[yi][l]; - //c = 1-cost*cost, = sint*sint - const DType cost = oout[i][0]/s; - const DType c = 1.0-cost*cost; - const DType dc_dx = -2.0/ss*oout[i][0]*w[yi][l]; - const DType d_sint_dc = 1.0/(2*sqrt(c)); - const DType d_sint_dx = dc_dx*d_sint_dc; - const DType df_dx = cos_m*w[yi][l] + s*nsin_m*d_sint_dx; - x_grad[i][l] += o_grad[i][yi] * (df_dx - w[yi][l]); - } - } -} - -template -__global__ void AmSoftmaxBackwardWKernel(const Tensor x, - const Tensor w, - const Tensor label, - const Tensor out, - const Tensor oout, - const Tensor o_grad, - Tensor w_grad, - const Tensor workspace, - const DType margin, - const DType s) { - const int nthreads = w.size(0) * w.size(1); - const int n = x.size(0); - const int feature_dim = w.size(1); - const DType cos_m = cos(margin); - const DType nsin_m = sin(margin)*-1.0; - const DType ss = s*s; - CUDA_KERNEL_LOOP(idx, nthreads) { - const int j = idx / feature_dim; - const int l = idx % feature_dim; - DType dw = 0; - for (int i = 0; i < n; ++i) { - const int yi = static_cast(label[i]); - if (yi == j&&oout[i][0]>=0.0) { - const DType cost = oout[i][0]/s; - const DType c = 1.0-cost*cost; - const DType dc_dw = -2.0/ss*oout[i][0]*x[i][l]; - const DType d_sint_dc = 1.0/(2*sqrt(c)); - const DType d_sint_dw = dc_dw*d_sint_dc; - const DType df_dw = cos_m*x[i][l] + s*nsin_m*d_sint_dw; - dw += o_grad[i][yi] * (df_dw - x[i][l]); - } - } - w_grad[j][l] += dw; - } -} - -template -inline void AmSoftmaxBackward(const Tensor &x, - const Tensor &w, - const Tensor &label, - const Tensor &out, - const Tensor &oout, - const Tensor &o_grad, - const Tensor &x_grad, - const Tensor &w_grad, - const Tensor &workspace, - const DType margin, - const DType s) { - const int n = x.size(0); - const int feature_dim = x.size(1); - const int m = w.size(0); - dim3 dimBlock(kBaseThreadNum); - dim3 dimGrid((n + kBaseThreadNum - 1) / kBaseThreadNum); - dimGrid.x = ((n * feature_dim + kBaseThreadNum - 1) / kBaseThreadNum); - AmSoftmaxBackwardXKernel<<>>(x, w, label, out, oout, o_grad, x_grad, workspace, - margin, s); - dimGrid.x = ((m * feature_dim + kBaseThreadNum - 1) / kBaseThreadNum); - AmSoftmaxBackwardWKernel<<>>(x, w, label, out, oout, o_grad, w_grad, workspace, - margin, s); -} - -} // namespace cuda - -template -inline void AmSoftmaxForward(const Tensor &x, - const Tensor &w, - const Tensor &label, - const Tensor &out, - const Tensor &oout, - const DType margin, - const DType s) { - cuda::AmSoftmaxForward(x, w, label, out, oout, margin, s); -} - -template -inline void AmSoftmaxBackward(const Tensor &x, - const Tensor &w, - const Tensor &label, - const Tensor &out, - const Tensor &oout, - const Tensor &o_grad, - const Tensor &x_grad, - const Tensor &w_grad, - const Tensor &workspace, - const DType margin, - const DType s) { - cuda::AmSoftmaxBackward(x, w, label, out, oout, o_grad, x_grad, w_grad, workspace, margin, s); -} - -} // namespace mshadow - -namespace mxnet { -namespace op { - -template<> -Operator *CreateOp(AmSoftmaxParam param, int dtype) { - Operator *op = NULL; - MSHADOW_REAL_TYPE_SWITCH(dtype, DType, { - op = new AmSoftmaxOp(param); - }) - return op; -} - -} // namespace op -} // namespace mxnet diff --git a/3rdparty/operator/lsoftmax-inl.h b/3rdparty/operator/lsoftmax-inl.h deleted file mode 100644 index a457eb4..0000000 --- a/3rdparty/operator/lsoftmax-inl.h +++ /dev/null @@ -1,377 +0,0 @@ -/*! - * Copyright (c) 2016 by Contributors - * \file lsoftmax-inl.h - * \brief LSoftmax from - * \author luoyetx - */ -#ifndef MXNET_OPERATOR_LSOFTMAX_INL_H_ -#define MXNET_OPERATOR_LSOFTMAX_INL_H_ - -#include -#include -#include -#include -#include -#include -#include -#include "./operator_common.h" - -namespace mxnet { -namespace op { - -namespace lsoftmax_enum { -enum LSoftmaxOpInputs {kData, kWeight, kLabel}; -enum LSoftmaxOpOutputs {kOut, kDataNorm, kWeightNorm}; -enum LSoftmaxResource {kTempSpace}; -} - -struct LSoftmaxParam : public dmlc::Parameter { - int margin; - float beta; - float beta_min; - float scale; - int num_hidden; - bool grad_norm; - int verbose; - float eps; - DMLC_DECLARE_PARAMETER(LSoftmaxParam) { - DMLC_DECLARE_FIELD(margin).set_default(2).set_lower_bound(1) - .describe("LSoftmax margin"); - DMLC_DECLARE_FIELD(beta).set_default(1).set_lower_bound(0) - .describe("LSoftmax beta, same as lambda to weight original value"); - DMLC_DECLARE_FIELD(beta_min).set_default(0).set_lower_bound(0) - .describe("Minimum beta"); - DMLC_DECLARE_FIELD(scale).set_default(1).set_range(0, 1) - .describe("Scale of beta during training for every iteration"); - DMLC_DECLARE_FIELD(num_hidden).set_lower_bound(1) - .describe("Number of hidden nodes of the output"); - DMLC_DECLARE_FIELD(grad_norm).set_default(false) - .describe("do grad norm"); - DMLC_DECLARE_FIELD(verbose).set_default(0) - .describe("Log for beta change"); - DMLC_DECLARE_FIELD(eps).set_default(1e-10f) - .describe("l2 eps"); - } -}; - -template -class LSoftmaxOp : public Operator { - public: - explicit LSoftmaxOp(LSoftmaxParam param) { - this->param_ = param; - // setup global lookup table - k_table_.clear(); - c_table_.clear(); - k_table_.push_back(1); - c_table_.push_back(1); - const int margin = param.margin; - const double pi = std::atan(1) * 4; - double factor = 1; - for (int i = 1; i <= margin; ++i) { - factor = factor * (margin - i + 1) / i; - k_table_.push_back(std::cos(i * pi / margin)); - c_table_.push_back(factor); - } - //next_beta_ = param.beta * 0.1f; - count_ = 0; - if(const char* env_p = std::getenv("BETA")) { - float _beta = std::atof(env_p); - if (param_.verbose) { - LOG(INFO)<<"beta:"<<_beta; - LOG(INFO)< &in_data, - const std::vector &req, - const std::vector &out_data, - const std::vector &aux_args) { - using namespace mshadow; - using namespace mshadow::expr; - CHECK_EQ(in_data.size(), 3); - CHECK_EQ(out_data.size(), 3); - CHECK_EQ(req.size(), 3); - CHECK_EQ(req[lsoftmax_enum::kOut], kWriteTo); - CHECK(req[lsoftmax_enum::kDataNorm] == kNullOp || - req[lsoftmax_enum::kDataNorm] == kWriteTo); - CHECK(req[lsoftmax_enum::kWeightNorm] == kNullOp || - req[lsoftmax_enum::kWeightNorm] == kWriteTo); - Stream *s = ctx.get_stream(); - const int n = in_data[lsoftmax_enum::kData].size(0); - const int m = in_data[lsoftmax_enum::kWeight].size(0); - Tensor x = in_data[lsoftmax_enum::kData].FlatTo2D(s); - Tensor w = in_data[lsoftmax_enum::kWeight].FlatTo2D(s); - Tensor label = in_data[lsoftmax_enum::kLabel].get_with_shape(Shape1(n), s); - Tensor out = out_data[lsoftmax_enum::kOut].FlatTo2D(s); - Tensor x_norm = out_data[lsoftmax_enum::kDataNorm].get_with_shape(Shape1(n), s); - Tensor w_norm = out_data[lsoftmax_enum::kWeightNorm].get_with_shape(Shape1(m), s); -#if defined(__CUDACC__) - CHECK_EQ(s->blas_handle_ownership_, Stream::OwnHandle) - << "Must init CuBLAS handle in stream"; -#endif - // original fully connected - out = dot(x, w.T()); - if (ctx.is_train) { - // large margin fully connected - const int margin = param_.margin; - if(const char* env_p = std::getenv("BETA")) { - float _beta = std::atof(env_p); - param_.beta = _beta; - } - const DType beta = static_cast(param_.beta); - //LOG(INFO)<<"beta:"< k_table_cpu(k_table_.data(), Shape1(k_table_.size())); - Tensor c_table_cpu(c_table_.data(), Shape1(c_table_.size())); - Tensor k_table_xpu(Shape1(k_table_.size())); - Tensor c_table_xpu(Shape1(c_table_.size())); - k_table_xpu.set_stream(s); - c_table_xpu.set_stream(s); - AllocSpace(&k_table_xpu); - AllocSpace(&c_table_xpu); - Copy(k_table_xpu, k_table_cpu, s); - Copy(c_table_xpu, c_table_cpu, s); - LSoftmaxForward(x, w, label, out, x_norm, w_norm, k_table_xpu, c_table_xpu, margin, beta); - FreeSpace(&k_table_xpu); - FreeSpace(&c_table_xpu); - } - } - - //virtual void GradNorm(mshadow::Tensor grad, mshadow::Stream* s) { - // using namespace mshadow; - // using namespace mshadow::expr; - // Tensor grad_cpu(grad.shape_); - // AllocSpace(&grad_cpu); - // Copy(grad_cpu, grad, s); - // DType grad_norm = param_.eps; - // for(uint32_t i=0;i grad, mshadow::Stream* s) { - using namespace mshadow; - using namespace mshadow::expr; - Tensor grad_cpu(grad.shape_); - AllocSpace(&grad_cpu); - Copy(grad_cpu, grad, s); - DType grad_norm = param_.eps; - for(uint32_t i=0;i &out_grad, - const std::vector &in_data, - const std::vector &out_data, - const std::vector &req, - const std::vector &in_grad, - const std::vector &aux_args) { - using namespace mshadow; - using namespace mshadow::expr; - CHECK_EQ(out_grad.size(), 1); - CHECK_EQ(in_data.size(), 3); - CHECK_EQ(out_data.size(), 3); - CHECK_GE(in_grad.size(), 2); - CHECK_GE(req.size(), 2); - CHECK_EQ(req[lsoftmax_enum::kData], kWriteTo); - CHECK_EQ(req[lsoftmax_enum::kWeight], kWriteTo); - Stream *s = ctx.get_stream(); - const int n = in_data[lsoftmax_enum::kData].size(0); - const int m = in_data[lsoftmax_enum::kWeight].size(0); - Tensor x = in_data[lsoftmax_enum::kData].FlatTo2D(s); - Tensor w = in_data[lsoftmax_enum::kWeight].FlatTo2D(s); - Tensor label = in_data[lsoftmax_enum::kLabel].get_with_shape(Shape1(n), s); - Tensor x_norm = out_data[lsoftmax_enum::kDataNorm].get_with_shape(Shape1(n), s); - Tensor w_norm = out_data[lsoftmax_enum::kWeightNorm].get_with_shape(Shape1(m), s); - Tensor o_grad = out_grad[lsoftmax_enum::kOut].FlatTo2D(s); - Tensor x_grad = in_grad[lsoftmax_enum::kData].FlatTo2D(s); - Tensor w_grad = in_grad[lsoftmax_enum::kWeight].FlatTo2D(s); - // workspace is used for cos_t, cos_mt, k, sin2_t, fo and cos_t_m for every data point - Tensor workspace = ctx.requested[lsoftmax_enum::kTempSpace].get_space_typed(Shape2(6, n), s); -#if defined(__CUDACC__) - CHECK_EQ(s->blas_handle_ownership_, Stream::OwnHandle) - << "Must init CuBLAS handle in stream"; -#endif - // original fully connected - x_grad = dot(o_grad, w); - w_grad = dot(o_grad.T(), x); - // large margin fully connected - const int margin = param_.margin; - const DType beta = static_cast(param_.beta); - count_+=1; - if (param_.verbose) { - if(count_%param_.verbose==0) { - LOG(INFO)<<"["< k_table_cpu(k_table_.data(), Shape1(k_table_.size())); - Tensor c_table_cpu(c_table_.data(), Shape1(c_table_.size())); - Tensor k_table_xpu(Shape1(k_table_.size())); - Tensor c_table_xpu(Shape1(c_table_.size())); - k_table_xpu.set_stream(s); - c_table_xpu.set_stream(s); - AllocSpace(&k_table_xpu); - AllocSpace(&c_table_xpu); - Copy(k_table_xpu, k_table_cpu, s); - Copy(c_table_xpu, c_table_cpu, s); - LSoftmaxBackward(x, w, label, x_norm, w_norm, o_grad, x_grad, w_grad, workspace, - k_table_xpu, c_table_xpu, margin, beta); - FreeSpace(&k_table_xpu); - FreeSpace(&c_table_xpu); - //if(param_.grad_norm) { - // GradNorm(x_grad, s); - // GradNorm(w_grad, s); - //} - // dirty hack, should also work for multi device - if(std::getenv("BETA")==NULL) { - param_.beta *= param_.scale; - param_.beta = std::max(param_.beta, param_.beta_min); - } - //LOG(INFO)<<"w_grad:"<(F(w_grad), 2); - //norm = F(norm + param_.eps); - //out = data / broadcast_with_axis(norm, 1, dshape[2]); - //if (param_.beta < next_beta_) { - // next_beta_ *= 0.1f; - // if (param_.verbose) { - // LOG(INFO) << "LSoftmax changes beta to " << param_.beta; - // } - //} - } - - //Tensor grad_norm(const Tensor grad) { - //} - - - - - private: - LSoftmaxParam param_; - // global lookup table - std::vector k_table_; - std::vector c_table_; - //float next_beta_; - uint32_t count_; -}; // class LSoftmaxOp - -template -Operator *CreateOp(LSoftmaxParam param, int dtype); - -#if DMLC_USE_CXX11 -class LSoftmaxProp : public OperatorProperty { - public: - void Init(const std::vector > &kwargs) override { - param_.Init(kwargs); - } - - std::map GetParams() const override { - return param_.__DICT__(); - } - - std::vector ListArguments() const override { - return {"data", "weight", "label"}; - } - - std::vector ListOutputs() const override { - return {"output", "data_norm", "weight_norm"}; - } - - int NumOutputs() const override { - return 3; - } - - int NumVisibleOutputs() const override { - return 1; - } - - bool InferShape(std::vector *in_shape, - std::vector *out_shape, - std::vector *aux_shape) const override { - using namespace mshadow; - CHECK_EQ(in_shape->size(), 3) << "Input:[data, label, weight]"; - const TShape &dshape = in_shape->at(lsoftmax_enum::kData); - const TShape &lshape = in_shape->at(lsoftmax_enum::kLabel); - CHECK_EQ(dshape.ndim(), 2) << "data shape should be (batch_size, feature_dim)"; - CHECK_EQ(lshape.ndim(), 1) << "label shape should be (batch_size,)"; - const int n = dshape[0]; - const int feature_dim = dshape[1]; - const int m = param_.num_hidden; - SHAPE_ASSIGN_CHECK(*in_shape, lsoftmax_enum::kWeight, Shape2(m, feature_dim)); - out_shape->clear(); - out_shape->push_back(Shape2(n, m)); // output - out_shape->push_back(Shape1(n)); // data norm - out_shape->push_back(Shape1(m)); // weight norm - aux_shape->clear(); - return true; - } - - std::vector BackwardResource( - const std::vector &in_shape) const override { - return {ResourceRequest::kTempSpace}; - } - - std::vector DeclareBackwardDependency( - const std::vector &out_grad, - const std::vector &in_data, - const std::vector &out_data) const override { - return {out_grad[lsoftmax_enum::kOut], out_data[lsoftmax_enum::kDataNorm], - out_data[lsoftmax_enum::kWeightNorm], in_data[lsoftmax_enum::kData], - in_data[lsoftmax_enum::kWeight], in_data[lsoftmax_enum::kLabel]}; - } - - std::string TypeString() const override { - return "LSoftmax"; - } - - OperatorProperty *Copy() const override { - auto ptr = new LSoftmaxProp(); - ptr->param_ = param_; - return ptr; - } - - Operator *CreateOperator(Context ctx) const override { - LOG(FATAL) << "Not Implemented."; - return NULL; - } - - Operator *CreateOperatorEx(Context ctx, std::vector *in_shape, - std::vector *in_type) const override; - - private: - LSoftmaxParam param_; -}; // class LSoftmaxProp -#endif // DMLC_USE_CXX11 - -} // namespace op -} // namespace mxnet - -#endif // MXNET_OPERATOR_LSOFTMAX_INL_H_ diff --git a/3rdparty/operator/lsoftmax.cc b/3rdparty/operator/lsoftmax.cc deleted file mode 100644 index cbf708b..0000000 --- a/3rdparty/operator/lsoftmax.cc +++ /dev/null @@ -1,75 +0,0 @@ -/*! - * Copyright (c) 2016 by Contributors - * \file lsoftmax.cc - * \brief LSoftmax from - * \author luoyetx - */ -#include "./lsoftmax-inl.h" - -namespace mshadow { - -template -inline void LSoftmaxForward(const Tensor &x, - const Tensor &w, - const Tensor &label, - const Tensor &out, - const Tensor &x_norm, - const Tensor &w_norm, - const Tensor &k_table, - const Tensor &c_table, - const int margin, - const DType beta) { - LOG(FATAL) << "Not Implemented."; -} - -template -inline void LSoftmaxBackward(const Tensor &x, - const Tensor &w, - const Tensor &label, - const Tensor &x_norm, - const Tensor &w_norm, - const Tensor &o_grad, - const Tensor &x_grad, - const Tensor &w_grad, - const Tensor &workspace, - const Tensor &k_table, - const Tensor &c_table, - const int margin, - const DType beta) { - LOG(FATAL) << "Not Implemented."; -} - -} // namespace mshadow - -namespace mxnet { -namespace op { - -template<> -Operator *CreateOp(LSoftmaxParam param, int dtype) { - Operator *op = NULL; - MSHADOW_REAL_TYPE_SWITCH(dtype, DType, { - op = new LSoftmaxOp(param); - }) - return op; -} - -Operator *LSoftmaxProp::CreateOperatorEx(Context ctx, std::vector *in_shape, - std::vector *in_type) const { - std::vector out_shape, aux_shape; - std::vector out_type, aux_type; - CHECK(InferType(in_type, &out_type, &aux_type)); - CHECK(InferShape(in_shape, &out_shape, &aux_shape)); - DO_BIND_DISPATCH(CreateOp, param_, in_type->at(0)); -} - -DMLC_REGISTER_PARAMETER(LSoftmaxParam); - -MXNET_REGISTER_OP_PROPERTY(LSoftmax, LSoftmaxProp) -.describe("LSoftmax from ") -.add_argument("data", "Symbol", "data") -.add_argument("weight", "Symbol", "weight") -.add_argument("label", "Symbol", "label") -.add_arguments(LSoftmaxParam::__FIELDS__()); - -} // namespace op -} // namespace mxnet diff --git a/3rdparty/operator/lsoftmax.cu b/3rdparty/operator/lsoftmax.cu deleted file mode 100644 index 6055c1b..0000000 --- a/3rdparty/operator/lsoftmax.cu +++ /dev/null @@ -1,322 +0,0 @@ -/*! - * Copyright (c) 2016 by Contributors - * \file lsoftmax.cu - * \brief LSoftmax from - * \author luoyetx - */ -#include "./lsoftmax-inl.h" - -namespace mshadow { -namespace cuda { - -namespace { -// workspace variables -enum LSoftmaxTempSpaceType {kCost, kCosmt, kK, kSin2t, kFo, kCostM}; -} - -#define CUDA_KERNEL_LOOP(i, n) \ - for (int i = blockIdx.x * blockDim.x + threadIdx.x; \ - i < (n); \ - i += blockDim.x * gridDim.x) - -MSHADOW_XINLINE int LSPowOfMO(const int k) { - return 1 - ((k&0x01) << 1); -} - -template -__global__ void LSCalcNorm(const Tensor x, - Tensor x_norm) { - const int n = x.size(0); - const int m = x.size(1); - CUDA_KERNEL_LOOP(i, n) { - DType norm = 0; - for (int j = 0; j < m; ++j) { - norm += x[i][j] * x[i][j]; - } - x_norm[i] = sqrt(norm); - } -} - -template -__device__ int LSFindK(const DType *k_table, const int n, const DType cos_t) { - const DType eps = 1e-5; - for (int i = 0; i < n; ++i) { - if (((k_table[i+1] < cos_t) || (abs(k_table[i+1] - cos_t) < eps)) && - ((k_table[i] > cos_t) || (abs(k_table[i] - cos_t) < eps))) { - return i; - } - } - return 0; -} - -template -__device__ DType LSCalcCosmt(const DType *c_table, const int n, - const DType cos_t, const int margin) { - const DType sin2_t = 1 - cos_t * cos_t; - DType cos_t_p = pow(cos_t, margin); - DType sin2_t_p = 1; - DType cos_mt = cos_t_p; // p = 0 - for (int p = 1; p <= margin / 2; ++p) { - cos_t_p /= cos_t * cos_t; // don't replace `cos_t*cos_t` with `1-sin2_t`, this can cause numeric issue if cos_t --> 0 - sin2_t_p *= sin2_t; - cos_mt += LSPowOfMO(p) * c_table[2*p] * cos_t_p * sin2_t_p; - } - return cos_mt; -} - -template -__global__ void LSoftmaxForwardKernel(const Tensor x, - const Tensor w, - const Tensor label, - const Tensor x_norm, - const Tensor w_norm, - Tensor out, - const Tensor k_table, - const Tensor c_table, - const int margin, - const DType beta) { - const int n = x.size(0); - const int feature_dim = x.size(1); - const int m = w.size(0); - CUDA_KERNEL_LOOP(i, n) { - const int yi = static_cast(label[i]); - const DType fo_i_yi = out[i][yi]; - const DType cos_t = fo_i_yi / (x_norm[i] * w_norm[yi]); - const int k = LSFindK(k_table.dptr_, k_table.size(0), cos_t); - const DType cos_mt = LSCalcCosmt(c_table.dptr_, c_table.size(0), cos_t, margin); - const DType f_i_yi = (LSPowOfMO(k) * cos_mt - 2*k) * (w_norm[yi] * x_norm[i]); - out[i][yi] = (f_i_yi + beta * fo_i_yi) / (1 + beta); - } -} - -template -inline void LSoftmaxForward(const Tensor &x, - const Tensor &w, - const Tensor &label, - const Tensor &out, - const Tensor &x_norm, - const Tensor &w_norm, - const Tensor &k_table, - const Tensor &c_table, - const int margin, - const DType beta) { - const int n = x.size(0); - const int m = w.size(0); - dim3 dimBlock(kBaseThreadNum); - dim3 dimGrid((n + kBaseThreadNum - 1) / kBaseThreadNum); - LSCalcNorm<<>>(x, x_norm); - dimGrid.x = ((m + kBaseThreadNum - 1) / kBaseThreadNum); - LSCalcNorm<<>>(w, w_norm); - dimGrid.x = ((n + kBaseThreadNum - 1) / kBaseThreadNum); - LSoftmaxForwardKernel<<>>(x, w, label, x_norm, w_norm, out, k_table, c_table, margin, beta); -} - -template -__global__ void LSoftmaxBackwardRequired(const Tensor x, - const Tensor w, - const Tensor label, - const Tensor x_norm, - const Tensor w_norm, - Tensor workspace, - const Tensor k_table, - const Tensor c_table, - const int margin) { - const int n = x.size(0); - const int feature_dim = x.size(1); - CUDA_KERNEL_LOOP(i, n) { - const int yi = static_cast(label[i]); - // fo_i_yi = dot(w_yi, x_i) - DType fo_i_yi = 0; - for (int p = 0; p < feature_dim; ++p) { - fo_i_yi += w[yi][p] * x[i][p]; - } - const DType cos_t = fo_i_yi / (x_norm[i] * w_norm[yi]); - const int k = LSFindK(k_table.dptr_, k_table.size(0), cos_t); - const DType cos_mt = LSCalcCosmt(c_table.dptr_, c_table.size(0), cos_t, margin); - const DType sin2_t = 1 - cos_t * cos_t; - workspace[kCost][i] = cos_t; - workspace[kCosmt][i] = cos_mt; - workspace[kK][i] = static_cast(k); - workspace[kSin2t][i] = sin2_t; - workspace[kFo][i] = fo_i_yi; - workspace[kCostM][i] = pow(cos_t, margin - 1); - } -} - -template -__global__ void LSoftmaxBackwardXKernel(const Tensor x, - const Tensor w, - const Tensor label, - const Tensor x_norm, - const Tensor w_norm, - const Tensor o_grad, - Tensor x_grad, - const Tensor workspace, - const Tensor c_table, - const int margin, - const DType beta) { - const int nthreads = x.size(0) * x.size(1); - const int feature_dim = x.size(1); - CUDA_KERNEL_LOOP(idx, nthreads) { - const int i = idx / feature_dim; - const int l = idx % feature_dim; - const int yi = static_cast(label[i]); - const DType cos_t = workspace[kCost][i]; - const DType cos_mt = workspace[kCosmt][i]; - const int k = static_cast(workspace[kK][i]); - const DType sin2_t = workspace[kSin2t][i]; - const DType fo_i_yi = workspace[kFo][i]; - const DType w_norm_yi = w_norm[yi]; - const DType x_norm_i = x_norm[i]; - - const DType dcos_dx = w[yi][l] / (w_norm_yi * x_norm_i) - \ - fo_i_yi * x[i][l] / (w_norm_yi * x_norm_i * x_norm_i * x_norm_i); - const DType dsin2_dx = -2 * cos_t * dcos_dx; - DType cos_t_p = workspace[kCostM][i]; - DType sin2_t_p = 1; - DType dcosm_dx = margin * cos_t_p * dcos_dx; // p = 0 - for (int p = 1; p <= margin / 2; ++p) { - cos_t_p /= cos_t * cos_t; - dcosm_dx += LSPowOfMO(p) * c_table[2*p] * (p * cos_t * dsin2_dx + \ - (margin - 2*p) * sin2_t * dcos_dx) * cos_t_p * sin2_t_p; - sin2_t_p *= sin2_t; - } - const DType df_dx = (LSPowOfMO(k) * cos_mt - 2*k) * w_norm_yi / x_norm_i * x[i][l] + \ - LSPowOfMO(k) * w_norm_yi * x_norm_i * dcosm_dx; - const DType alpha = 1 / (1 + beta); - x_grad[i][l] += alpha * o_grad[i][yi] * (df_dx - w[yi][l]); - } -} - -template -__global__ void LSoftmaxBackwardWKernel(const Tensor x, - const Tensor w, - const Tensor label, - const Tensor x_norm, - const Tensor w_norm, - const Tensor o_grad, - Tensor w_grad, - const Tensor workspace, - const Tensor c_table, - const int margin, - const DType beta) { - const int nthreads = w.size(0) * w.size(1); - const int n = x.size(0); - const int feature_dim = w.size(1); - CUDA_KERNEL_LOOP(idx, nthreads) { - const int j = idx / feature_dim; - const int l = idx % feature_dim; - DType dw = 0; - for (int i = 0; i < n; ++i) { - const int yi = static_cast(label[i]); - if (yi == j) { - const DType cos_t = workspace[kCost][i]; - const DType cos_mt = workspace[kCosmt][i]; - const int k = static_cast(workspace[kK][i]); - const DType sin2_t = workspace[kSin2t][i]; - const DType fo_i_yi = workspace[kFo][i]; - const DType x_norm_i = x_norm[i]; - const DType w_norm_yi = w_norm[yi]; - - const DType dcos_dw = x[i][l] / (w_norm_yi * x_norm_i) - \ - fo_i_yi * w[yi][l] / (x_norm_i * w_norm_yi * w_norm_yi * w_norm_yi); - const DType dsin2_dw = -2 * cos_t * dcos_dw; - DType cos_t_p = workspace[kCostM][i]; - DType sin2_t_p = 1; - DType dcosm_dw = margin * cos_t_p * dcos_dw; // p = 0 - for (int p = 1; p <= margin / 2; ++p) { - cos_t_p /= cos_t * cos_t; - dcosm_dw += LSPowOfMO(p) * c_table[2*p] * (p * cos_t * dsin2_dw + \ - (margin - 2*p) * sin2_t * dcos_dw) * cos_t_p * sin2_t_p; - sin2_t_p *= sin2_t; - } - const DType df_dw_j = (LSPowOfMO(k) * cos_mt - 2*k) * x_norm_i / w_norm_yi * w[yi][l] + \ - LSPowOfMO(k) * w_norm_yi * x_norm_i * dcosm_dw; - dw += o_grad[i][yi] * (df_dw_j - x[i][l]); - } - } - const DType alpha = 1 / (1 + beta); - w_grad[j][l] += alpha * dw; - } -} - -template -inline void LSoftmaxBackward(const Tensor &x, - const Tensor &w, - const Tensor &label, - const Tensor &x_norm, - const Tensor &w_norm, - const Tensor &o_grad, - const Tensor &x_grad, - const Tensor &w_grad, - const Tensor &workspace, - const Tensor &k_table, - const Tensor &c_table, - const int margin, - const DType beta) { - const int n = x.size(0); - const int feature_dim = x.size(1); - const int m = w.size(0); - dim3 dimBlock(kBaseThreadNum); - dim3 dimGrid((n + kBaseThreadNum - 1) / kBaseThreadNum); - LSoftmaxBackwardRequired<<>>(x, w, label, x_norm, w_norm, workspace, - k_table, c_table, margin); - dimGrid.x = ((n * feature_dim + kBaseThreadNum - 1) / kBaseThreadNum); - LSoftmaxBackwardXKernel<<>>(x, w, label, x_norm, w_norm, o_grad, x_grad, workspace, - c_table, margin, beta); - dimGrid.x = ((m * feature_dim + kBaseThreadNum - 1) / kBaseThreadNum); - LSoftmaxBackwardWKernel<<>>(x, w, label, x_norm, w_norm, o_grad, w_grad, workspace, - c_table, margin, beta); -} - -} // namespace cuda - -template -inline void LSoftmaxForward(const Tensor &x, - const Tensor &w, - const Tensor &label, - const Tensor &out, - const Tensor &x_norm, - const Tensor &w_norm, - const Tensor &k_table, - const Tensor &c_table, - const int margin, - const DType beta) { - cuda::LSoftmaxForward(x, w, label, out, x_norm, w_norm, - k_table, c_table, margin, beta); -} - -template -inline void LSoftmaxBackward(const Tensor &x, - const Tensor &w, - const Tensor &label, - const Tensor &x_norm, - const Tensor &w_norm, - const Tensor &o_grad, - const Tensor &x_grad, - const Tensor &w_grad, - const Tensor &workspace, - const Tensor &k_table, - const Tensor &c_table, - const int margin, - const DType beta) { - cuda::LSoftmaxBackward(x, w, label, x_norm, w_norm, o_grad, x_grad, w_grad, workspace, - k_table, c_table, margin, beta); -} - -} // namespace mshadow - -namespace mxnet { -namespace op { - -template<> -Operator *CreateOp(LSoftmaxParam param, int dtype) { - Operator *op = NULL; - MSHADOW_REAL_TYPE_SWITCH(dtype, DType, { - op = new LSoftmaxOp(param); - }) - return op; -} - -} // namespace op -} // namespace mxnet diff --git a/Evaluation/IJB/IJB_1N.py b/Evaluation/IJB/IJB_1N.py deleted file mode 100644 index e42b49b..0000000 --- a/Evaluation/IJB/IJB_1N.py +++ /dev/null @@ -1,314 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 -import os -import numpy as np -import timeit -import sklearn -import cv2 -import sys -import argparse -import glob -import numpy.matlib -import heapq -import math -from datetime import datetime as dt - -from sklearn import preprocessing -sys.path.append('./recognition') -from embedding import Embedding -from menpo.visualize import print_progress -from menpo.visualize.viewmatplotlib import sample_colours_from_colourmap - -def read_template_subject_id_list(path): - ijb_meta = np.loadtxt(path, dtype=str, skiprows=1, delimiter=',') - templates = ijb_meta[:, 0].astype(np.int) - subject_ids = ijb_meta[:, 1].astype(np.int) - return templates, subject_ids - -def read_template_media_list(path): - ijb_meta = np.loadtxt(path, dtype=str) - templates = ijb_meta[:,1].astype(np.int) - medias = ijb_meta[:,2].astype(np.int) - return templates, medias - -def read_template_pair_list(path): - pairs = np.loadtxt(path, dtype=str) - t1 = pairs[:,0].astype(np.int) - t2 = pairs[:,1].astype(np.int) - label = pairs[:,2].astype(np.int) - return t1, t2, label - - -#def get_image_feature(feature_path, faceness_path): -# img_feats = np.loadtxt(feature_path) -# faceness_scores = np.loadtxt(faceness_path) -# return img_feats, faceness_scores -def get_image_feature(img_path, img_list_path, model_path, epoch, gpu_id): - img_list = open(img_list_path) - embedding = Embedding(model_path, epoch, gpu_id) - files = img_list.readlines() - print('files:', len(files)) - faceness_scores = [] - img_feats = [] - for img_index, each_line in enumerate(files): - if img_index%500==0: - print('processing', img_index) - name_lmk_score = each_line.strip().split(' ') - img_name = os.path.join(img_path, name_lmk_score[0]) - img = cv2.imread(img_name) - lmk = np.array([float(x) for x in name_lmk_score[1:-1]], dtype=np.float32) - lmk = lmk.reshape( (5,2) ) - img_feats.append(embedding.get(img,lmk)) - faceness_scores.append(name_lmk_score[-1]) - img_feats = np.array(img_feats).astype(np.float32) - faceness_scores = np.array(faceness_scores).astype(np.float32) - - #img_feats = np.ones( (len(files), 1024), dtype=np.float32) * 0.01 - #faceness_scores = np.ones( (len(files), ), dtype=np.float32 ) - return img_feats, faceness_scores - -def image2template_feature(img_feats = None, templates = None, medias = None, choose_templates = None, choose_ids = None): - # ========================================================== - # 1. face image feature l2 normalization. img_feats:[number_image x feats_dim] - # 2. compute media feature. - # 3. compute template feature. - # ========================================================== - unique_templates, indices = np.unique(choose_templates, return_index=True) - unique_subjectids = choose_ids[indices] - template_feats = np.zeros((len(unique_templates), img_feats.shape[1])) - - for count_template, uqt in enumerate(unique_templates): - (ind_t,) = np.where(templates == uqt) - face_norm_feats = img_feats[ind_t] - face_medias = medias[ind_t] - unique_medias, unique_media_counts = np.unique(face_medias, return_counts=True) - media_norm_feats = [] - for u,ct in zip(unique_medias, unique_media_counts): - (ind_m,) = np.where(face_medias == u) - if ct == 1: - media_norm_feats += [face_norm_feats[ind_m]] - else: # image features from the same video will be aggregated into one feature - media_norm_feats += [np.mean(face_norm_feats[ind_m], 0, keepdims=True)] - media_norm_feats = np.array(media_norm_feats) - # media_norm_feats = media_norm_feats / np.sqrt(np.sum(media_norm_feats ** 2, -1, keepdims=True)) - template_feats[count_template] = np.sum(media_norm_feats, 0) - if count_template % 2000 == 0: - print('Finish Calculating {} template features.'.format(count_template)) - template_norm_feats = template_feats / np.sqrt(np.sum(template_feats ** 2, -1, keepdims=True)) - return template_norm_feats, unique_templates, unique_subjectids - -def verification(template_norm_feats = None, unique_templates = None, p1 = None, p2 = None): - # ========================================================== - # Compute set-to-set Similarity Score. - # ========================================================== - template2id = np.zeros((max(unique_templates)+1,1),dtype=int) - for count_template, uqt in enumerate(unique_templates): - template2id[uqt] = count_template - - score = np.zeros((len(p1),)) # save cosine distance between pairs - - total_pairs = np.array(range(len(p1))) - batchsize = 100000 # small batchsize instead of all pairs in one batch due to the memory limiation - sublists = [total_pairs[i:i + batchsize] for i in range(0, len(p1), batchsize)] - total_sublists = len(sublists) - for c, s in enumerate(sublists): - feat1 = template_norm_feats[template2id[p1[s]]] - feat2 = template_norm_feats[template2id[p2[s]]] - similarity_score = np.sum(feat1 * feat2, -1) - score[s] = similarity_score.flatten() - if c % 10 == 0: - print('Finish {}/{} pairs.'.format(c, total_sublists)) - return score - -def read_score(path): - with open(path, 'rb') as fid: - img_feats = cPickle.load(fid) - return img_feats - -def evaluation(query_feats, gallery_feats, mask): - Fars = [0.01, 0.1] - print(query_feats.shape) - print(gallery_feats.shape) - - query_num = query_feats.shape[0] - gallery_num = gallery_feats.shape[0] - - similarity = np.dot(query_feats, gallery_feats.T) - print('similarity shape', similarity.shape) - top_inds = np.argsort(-similarity) - print(top_inds.shape) - - # calculate top1 - correct_num = 0 - for i in range(query_num): - j = top_inds[i, 0] - if j == mask[i]: - correct_num += 1 - print("top1 = {}".format(correct_num/query_num)) - # calculate top5 - correct_num = 0 - for i in range(query_num): - j = top_inds[i, 0:5] - if mask[i] in j: - correct_num += 1 - print("top5 = {}".format(correct_num/query_num)) - # calculate 10 - correct_num = 0 - for i in range(query_num): - j = top_inds[i, 0:10] - if mask[i] in j: - correct_num += 1 - print("top10 = {}".format(correct_num/query_num)) - - neg_pair_num = query_num * gallery_num - query_num - print(neg_pair_num) - required_topk = [math.ceil(query_num * x) for x in Fars] - top_sims = similarity - # calculate fars and tprs - pos_sims = [] - for i in range(query_num): - gt = mask[i] - pos_sims.append(top_sims[i, gt]) - top_sims[i, gt] = -2.0 - - pos_sims = np.array(pos_sims) - print(pos_sims.shape) - neg_sims = top_sims[np.where(top_sims > -2.0)] - print("neg_sims num = {}".format(len(neg_sims))) - neg_sims = heapq.nlargest(max(required_topk), neg_sims) # heap sort - print("after sorting , neg_sims num = {}".format(len(neg_sims))) - for far, pos in zip(Fars, required_topk): - th = neg_sims[pos-1] - recall = np.sum(pos_sims > th) / query_num - print("far = {:.10f} pr = {:.10f} th = {:.10f}".format(far, recall, th)) - -def gen_mask(query_ids, reg_ids): - mask = [] - for query_id in query_ids: - pos = [i for i, x in enumerate(reg_ids) if query_id == x] - if len(pos) != 1: - raise RuntimeError("RegIdsError with id = {}， duplicate = {} ".format(query_id, len(pos))) - mask.append(pos[0]) - return mask - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description='do ijb 1n test') - # general - parser.add_argument('--model-prefix', default='', help='path to load model.') - parser.add_argument('--model-epoch', default=1, type=int, help='') - parser.add_argument('--gpu', default=7, type=int, help='gpu id') - parser.add_argument('--batch-size', default=32, type=int, help='') - parser.add_argument('--job', default='insightface', type=str, help='job name') - parser.add_argument('--target', default='IJBC', type=str, help='target, set to IJBC or IJBB') - args = parser.parse_args() - target = args.target - model_path = args.model_prefix - gpu_id = args.gpu - epoch = args.model_epoch - meta_dir = "%s/meta"%args.target #meta root dir - if target=='IJBC': - gallery_s1_record = "%s_1N_gallery_G1.csv" % (args.target.lower()) - gallery_s2_record = "%s_1N_gallery_G2.csv" % (args.target.lower()) - else: - gallery_s1_record = "%s_1N_gallery_S1.csv" % (args.target.lower()) - gallery_s2_record = "%s_1N_gallery_S2.csv" % (args.target.lower()) - gallery_s1_templates, gallery_s1_subject_ids = read_template_subject_id_list(os.path.join(meta_dir, gallery_s1_record)) - print(gallery_s1_templates.shape, gallery_s1_subject_ids.shape) - - gallery_s2_templates, gallery_s2_subject_ids = read_template_subject_id_list(os.path.join(meta_dir, gallery_s2_record)) - print(gallery_s2_templates.shape, gallery_s2_templates.shape) - - gallery_templates = np.concatenate([gallery_s1_templates, gallery_s2_templates]) - gallery_subject_ids = np.concatenate([gallery_s1_subject_ids, gallery_s2_subject_ids]) - print(gallery_templates.shape, gallery_subject_ids.shape) - - media_record = "%s_face_tid_mid.txt" % args.target.lower() - total_templates, total_medias = read_template_media_list(os.path.join(meta_dir, media_record)) - print("total_templates", total_templates.shape, total_medias.shape) - #load image features - start = timeit.default_timer() - feature_path = '' #feature path - face_path = '' #face path - img_path = './%s/loose_crop' % target - img_list_path = './%s/meta/%s_name_5pts_score.txt' % (target, target.lower()) - #img_feats, faceness_scores = get_image_feature(feature_path, face_path) - img_feats, faceness_scores = get_image_feature(img_path, img_list_path, model_path, epoch, gpu_id) - print('img_feats', img_feats.shape) - print('faceness_scores', faceness_scores.shape) - stop = timeit.default_timer() - print('Time: %.2f s. ' % (stop - start)) - print('Feature Shape: ({} , {}) .'.format(img_feats.shape[0], img_feats.shape[1])) - - # compute template features from image features. - start = timeit.default_timer() - # ========================================================== - # Norm feature before aggregation into template feature? - # Feature norm from embedding network and faceness score are able to decrease weights for noise samples (not face). - # ========================================================== - use_norm_score = True # if True, TestMode(N1) - use_detector_score = True # if True, TestMode(D1) - use_flip_test = True # if True, TestMode(F1) - - if use_flip_test: - # concat --- F1 - #img_input_feats = img_feats - # add --- F2 - img_input_feats = img_feats[:,0:int(img_feats.shape[1]/2)] + img_feats[:,int(img_feats.shape[1]/2):] - else: - img_input_feats = img_feats[:,0:int(img_feats.shape[1]/2)] - - if use_norm_score: - img_input_feats = img_input_feats - else: - # normalise features to remove norm information - img_input_feats = img_input_feats / np.sqrt(np.sum(img_input_feats ** 2, -1, keepdims=True)) - - if use_detector_score: - img_input_feats = img_input_feats * np.matlib.repmat(faceness_scores[:,np.newaxis], 1, img_input_feats.shape[1]) - else: - img_input_feats = img_input_feats - print("input features shape", img_input_feats.shape) - - #load gallery feature - gallery_templates_feature, gallery_unique_templates, gallery_unique_subject_ids = image2template_feature(img_input_feats, total_templates, total_medias, gallery_templates, gallery_subject_ids) - stop = timeit.default_timer() - print('Time: %.2f s. ' % (stop - start)) - print("gallery_templates_feature", gallery_templates_feature.shape) - print("gallery_unique_subject_ids", gallery_unique_subject_ids.shape) - #np.savetxt("gallery_templates_feature.txt", gallery_templates_feature) - #np.savetxt("gallery_unique_subject_ids.txt", gallery_unique_subject_ids) - - #load prope feature - probe_mixed_record = "%s_1N_probe_mixed.csv" % target.lower() - probe_mixed_templates, probe_mixed_subject_ids = read_template_subject_id_list(os.path.join(meta_dir, probe_mixed_record)) - print(probe_mixed_templates.shape, probe_mixed_subject_ids.shape) - probe_mixed_templates_feature, probe_mixed_unique_templates, probe_mixed_unique_subject_ids = image2template_feature(img_input_feats, total_templates, total_medias, probe_mixed_templates, probe_mixed_subject_ids) - print("probe_mixed_templates_feature", probe_mixed_templates_feature.shape) - print("probe_mixed_unique_subject_ids", probe_mixed_unique_subject_ids.shape) - #np.savetxt("probe_mixed_templates_feature.txt", probe_mixed_templates_feature) - #np.savetxt("probe_mixed_unique_subject_ids.txt", probe_mixed_unique_subject_ids) - - #root_dir = "" #feature root dir - #gallery_id_path = "" #id filepath - #gallery_feats_path = "" #feature filelpath - #print("{}: start loading gallery feat {}".format(dt.now(), gallery_id_path)) - #gallery_ids, gallery_feats = load_feat_file(root_dir, gallery_id_path, gallery_feats_path) - #print("{}: end loading gallery feat".format(dt.now())) - # - #probe_id_path = "probe_mixed_unique_subject_ids.txt" #probe id filepath - #probe_feats_path = "probe_mixed_templates_feature.txt" #probe feats filepath - #print("{}: start loading probe feat {}".format(dt.now(), probe_id_path)) - #probe_ids, probe_feats = load_feat_file(root_dir, probe_id_path, probe_feats_path) - #print("{}: end loading probe feat".format(dt.now())) - - gallery_ids = gallery_unique_subject_ids - gallery_feats = gallery_templates_feature - probe_ids = probe_mixed_unique_subject_ids - probe_feats = probe_mixed_templates_feature - - mask = gen_mask(probe_ids, gallery_ids) - - print("{}: start evaluation".format(dt.now())) - evaluation(probe_feats, gallery_feats, mask) - print("{}: end evaluation".format(dt.now())) - diff --git a/Evaluation/Megaface/gen_megaface.py b/Evaluation/Megaface/gen_megaface.py deleted file mode 100644 index 6ec1333..0000000 --- a/Evaluation/Megaface/gen_megaface.py +++ /dev/null @@ -1,179 +0,0 @@ - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os - -from easydict import EasyDict as edict -import time -import sys -import numpy as np -import argparse -import struct -import cv2 -import sklearn -from sklearn.preprocessing import normalize -import mxnet as mx -from mxnet import ndarray as nd - - - - -def read_img(image_path): - img = cv2.imread(image_path, cv2.CV_LOAD_IMAGE_COLOR) - return img - -def get_feature(imgs, nets): - count = len(imgs) - data = mx.nd.zeros(shape = (count*2, 3, imgs[0].shape[0], imgs[0].shape[1])) - for idx, img in enumerate(imgs): - img = img[:,:,::-1] #to rgb - img = np.transpose( img, (2,0,1) ) - for flipid in [0,1]: - _img = np.copy(img) - if flipid==1: - _img = _img[:,:,::-1] - _img = nd.array(_img) - data[count*flipid+idx] = _img - - F = [] - for net in nets: - db = mx.io.DataBatch(data=(data,)) - net.model.forward(db, is_train=False) - x = net.model.get_outputs()[0].asnumpy() - embedding = x[0:count,:] + x[count:,:] - embedding = sklearn.preprocessing.normalize(embedding) - #print('emb', embedding.shape) - F.append(embedding) - F = np.concatenate(F, axis=1) - F = sklearn.preprocessing.normalize(F) - #print('F', F.shape) - return F - - -def write_bin(path, feature): - feature = list(feature) - with open(path, 'wb') as f: - f.write(struct.pack('4i', len(feature),1,4,5)) - f.write(struct.pack("%df"%len(feature), *feature)) - -def get_and_write(buffer, nets): - imgs = [] - for k in buffer: - imgs.append(k[0]) - features = get_feature(imgs, nets) - #print(np.linalg.norm(feature)) - assert features.shape[0]==len(buffer) - for ik,k in enumerate(buffer): - out_path = k[1] - feature = features[ik].flatten() - write_bin(out_path, feature) - -def main(args): - - print(args) - gpuid = args.gpu - ctx = mx.gpu(gpuid) - nets = [] - image_shape = [int(x) for x in args.image_size.split(',')] - for model in args.model.split('|'): - vec = model.split(',') - assert len(vec)>1 - prefix = vec[0] - epoch = int(vec[1]) - print('loading',prefix, epoch) - net = edict() - net.ctx = ctx - net.sym, net.arg_params, net.aux_params = mx.model.load_checkpoint(prefix, epoch) - all_layers = net.sym.get_internals() - net.sym = all_layers['fc1_output'] - net.model = mx.mod.Module(symbol=net.sym, context=net.ctx, label_names = None) - net.model.bind(data_shapes=[('data', (1, 3, image_shape[1], image_shape[2]))]) - net.model.set_params(net.arg_params, net.aux_params) - nets.append(net) - - facescrub_out = os.path.join(args.output, 'facescrub') - megaface_out = os.path.join(args.output, 'megaface') - - i = 0 - succ = 0 - buffer = [] - for line in open(args.facescrub_lst, 'r'): - if i%1000==0: - print("writing fs",i, succ) - i+=1 - image_path = line.strip() - _path = image_path.split('/') - a,b = _path[-2], _path[-1] - out_dir = os.path.join(facescrub_out, a) - if not os.path.exists(out_dir): - os.makedirs(out_dir) - image_path = os.path.join(args.facescrub_root, image_path) - img = read_img(image_path) - if img is None: - print('read error:', image_path) - continue - out_path = os.path.join(out_dir, b+"_%s.bin"%(args.algo)) - item = (img, out_path) - buffer.append(item) - if len(buffer)==args.batch_size: - get_and_write(buffer, nets) - buffer = [] - succ+=1 - if len(buffer)>0: - get_and_write(buffer, nets) - buffer = [] - print('fs stat',i, succ) - - i = 0 - succ = 0 - buffer = [] - for line in open(args.megaface_lst, 'r'): - if i%1000==0: - print("writing mf",i, succ) - i+=1 - image_path = line.strip() - _path = image_path.split('/') - a1, a2, b = _path[-3], _path[-2], _path[-1] - out_dir = os.path.join(megaface_out, a1, a2) - if not os.path.exists(out_dir): - os.makedirs(out_dir) - #continue - #print(landmark) - image_path = os.path.join(args.megaface_root, image_path) - img = read_img(image_path) - if img is None: - print('read error:', image_path) - continue - out_path = os.path.join(out_dir, b+"_%s.bin"%(args.algo)) - item = (img, out_path) - buffer.append(item) - if len(buffer)==args.batch_size: - get_and_write(buffer, nets) - buffer = [] - succ+=1 - if len(buffer)>0: - get_and_write(buffer, nets) - buffer = [] - print('mf stat',i, succ) - - -def parse_arguments(argv): - parser = argparse.ArgumentParser() - - parser.add_argument('--batch_size', type=int, help='', default=8) - parser.add_argument('--image_size', type=str, help='', default='3,112,112') - parser.add_argument('--gpu', type=int, help='', default=0) - parser.add_argument('--algo', type=str, help='', default='insightface') - parser.add_argument('--facescrub-lst', type=str, help='', default='./data/facescrub_lst') - parser.add_argument('--megaface-lst', type=str, help='', default='./data/megaface_lst') - parser.add_argument('--facescrub-root', type=str, help='', default='./data/facescrub_images') - parser.add_argument('--megaface-root', type=str, help='', default='./data/megaface_images') - parser.add_argument('--output', type=str, help='', default='./feature_out') - parser.add_argument('--model', type=str, help='', default='') - return parser.parse_args(argv) - -if __name__ == '__main__': - main(parse_arguments(sys.argv[1:])) - diff --git a/Evaluation/Megaface/remove_noises.py b/Evaluation/Megaface/remove_noises.py deleted file mode 100644 index f80f350..0000000 --- a/Evaluation/Megaface/remove_noises.py +++ /dev/null @@ -1,156 +0,0 @@ - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import datetime -import time -import shutil -import sys -import numpy as np -import argparse -import struct -import cv2 -import mxnet as mx -from mxnet import ndarray as nd - - -feature_dim = 512 -feature_ext = 1 - -def load_bin(path, fill = 0.0): - with open(path, 'rb') as f: - bb = f.read(4*4) - #print(len(bb)) - v = struct.unpack('4i', bb) - #print(v[0]) - bb = f.read(v[0]*4) - v = struct.unpack("%df"%(v[0]), bb) - feature = np.full( (feature_dim+feature_ext,), fill, dtype=np.float32) - feature[0:feature_dim] = v - #feature = np.array( v, dtype=np.float32) - #print(feature.shape) - #print(np.linalg.norm(feature)) - return feature - -def write_bin(path, feature): - feature = list(feature) - with open(path, 'wb') as f: - f.write(struct.pack('4i', len(feature),1,4,5)) - f.write(struct.pack("%df"%len(feature), *feature)) - -def main(args): - - - fs_noise_map = {} - for line in open(args.facescrub_noises, 'r'): - if line.startswith('#'): - continue - line = line.strip() - fname = line.split('.')[0] - p = fname.rfind('_') - fname = fname[0:p] - fs_noise_map[line] = fname - - print(len(fs_noise_map)) - - i=0 - fname2center = {} - noises = [] - for line in open(args.facescrub_lst, 'r'): - if i%1000==0: - print("reading fs",i) - i+=1 - image_path = line.strip() - _path = image_path.split('/') - a, b = _path[-2], _path[-1] - feature_path = os.path.join(args.feature_dir_input, 'facescrub', a, "%s_%s.bin"%(b, args.algo)) - feature_dir_out = os.path.join(args.feature_dir_out, 'facescrub', a) - if not os.path.exists(feature_dir_out): - os.makedirs(feature_dir_out) - feature_path_out = os.path.join(feature_dir_out, "%s_%s.bin"%(b, args.algo)) - #print(b) - if not b in fs_noise_map: - #shutil.copyfile(feature_path, feature_path_out) - feature = load_bin(feature_path) - write_bin(feature_path_out, feature) - if not a in fname2center: - fname2center[a] = np.zeros((feature_dim+feature_ext,), dtype=np.float32) - fname2center[a] += feature - else: - #print('n', b) - noises.append( (a,b) ) - print(len(noises)) - - for k in noises: - a,b = k - assert a in fname2center - center = fname2center[a] - g = np.zeros( (feature_dim+feature_ext,), dtype=np.float32) - g2 = np.random.uniform(-0.001, 0.001, (feature_dim,)) - g[0:feature_dim] = g2 - f = center+g - _norm=np.linalg.norm(f) - f /= _norm - feature_path_out = os.path.join(args.feature_dir_out, 'facescrub', a, "%s_%s.bin"%(b, args.algo)) - write_bin(feature_path_out, f) - - mf_noise_map = {} - for line in open(args.megaface_noises, 'r'): - if line.startswith('#'): - continue - line = line.strip() - _vec = line.split("\t") - if len(_vec)>1: - line = _vec[1] - mf_noise_map[line] = 1 - - print(len(mf_noise_map)) - - i=0 - nrof_noises = 0 - for line in open(args.megaface_lst, 'r'): - if i%1000==0: - print("reading mf",i) - i+=1 - image_path = line.strip() - _path = image_path.split('/') - a1, a2, b = _path[-3], _path[-2], _path[-1] - feature_path = os.path.join(args.feature_dir_input, 'megaface', a1, a2, "%s_%s.bin"%(b, args.algo)) - feature_dir_out = os.path.join(args.feature_dir_out, 'megaface', a1, a2) - if not os.path.exists(feature_dir_out): - os.makedirs(feature_dir_out) - feature_path_out = os.path.join(feature_dir_out, "%s_%s.bin"%(b, args.algo)) - bb = '/'.join([a1, a2, b]) - #print(b) - if not bb in mf_noise_map: - feature = load_bin(feature_path) - write_bin(feature_path_out, feature) - #shutil.copyfile(feature_path, feature_path_out) - else: - feature = load_bin(feature_path, 100.0) - write_bin(feature_path_out, feature) - #g = np.random.uniform(-0.001, 0.001, (feature_dim,)) - #print('n', bb) - #write_bin(feature_path_out, g) - nrof_noises+=1 - print(nrof_noises) - - -def parse_arguments(argv): - parser = argparse.ArgumentParser() - - parser.add_argument('--facescrub-noises', type=str, help='', default='./data/facescrub_noises.txt') - parser.add_argument('--megaface-noises', type=str, help='', default='./data/megaface_noises.txt') - parser.add_argument('--algo', type=str, help='', default='insightface') - parser.add_argument('--facescrub-lst', type=str, help='', default='./data/facescrub_lst') - parser.add_argument('--megaface-lst', type=str, help='', default='./data/megaface_lst') - parser.add_argument('--feature-dir-input', type=str, help='', default='./feature_out') - parser.add_argument('--feature-dir-out', type=str, help='', default='./feature_out_clean') - return parser.parse_args(argv) - -if __name__ == '__main__': - main(parse_arguments(sys.argv[1:])) - - diff --git a/PRNet.mxnet/README.md b/PRNet.mxnet/README.md deleted file mode 100644 index a9896bf..0000000 --- a/PRNet.mxnet/README.md +++ /dev/null @@ -1,6 +0,0 @@ -MXNet implementation of [Joint 3D Face Reconstruction and Dense Alignment with Position Map Regression Network](http://openaccess.thecvf.com/content_ECCV_2018/papers/Yao_Feng_Joint_3D_Face_ECCV_2018_paper.pdf). - -Original [PyTorch implementation](https://github.com/YadiraF/PRNet) - -Pretrained Models and details coming soon. - diff --git a/PRNet.mxnet/config.py b/PRNet.mxnet/config.py deleted file mode 100644 index 7ff5ba9..0000000 --- a/PRNet.mxnet/config.py +++ /dev/null @@ -1,89 +0,0 @@ -import numpy as np -from easydict import EasyDict as edict - -config = edict() - -#default training/dataset config -config.num_classes = 3 -config.input_img_size = 256 -config.output_label_size = 64 - -# network settings -network = edict() - -network.hourglass = edict() -network.hourglass.net_sta = 0 -network.hourglass.net_n = 4 -network.hourglass.net_dcn = 0 -network.hourglass.net_stacks = 1 -network.hourglass.net_block = 'resnet' -network.hourglass.net_binarize = False -network.hourglass.losstype = 'heatmap' -network.hourglass.multiplier = 1.0 - -network.prnet = edict() -network.prnet.net_sta = 0 -network.prnet.net_n = 5 -network.prnet.net_dcn = 0 -network.prnet.net_stacks = 1 -network.prnet.net_modules = 2 -network.prnet.net_block = 'hpm' -network.prnet.net_binarize = False -network.prnet.losstype = 'heatmap' -network.prnet.multiplier = 0.25 - -network.hpm = edict() -network.hpm.net_sta = 0 -network.hpm.net_n = 4 -network.hpm.net_dcn = 0 -network.hpm.net_stacks = 1 -network.hpm.net_block = 'hpm' -network.hpm.net_binarize = False -network.hpm.losstype = 'heatmap' -network.hpm.multiplier = 1.0 - - -# dataset settings -dataset = edict() - - -dataset.prnet = edict() -dataset.prnet.dataset = '3D' -dataset.prnet.landmark_type = 'dense' -dataset.prnet.dataset_path = './data64' -dataset.prnet.num_classes = 3 -dataset.prnet.input_img_size = 256 -dataset.prnet.output_label_size = 64 -#dataset.prnet.label_xfirst = False -dataset.prnet.val_targets = [''] - -# default settings -default = edict() - -# default network -default.network = 'hpm' -default.pretrained = '' -default.pretrained_epoch = 0 -# default dataset -default.dataset = 'prnet' -default.frequent = 20 -default.verbose = 200 -default.kvstore = 'device' - -default.prefix = 'model/A' -default.end_epoch = 10000 -default.lr = 0.00025 -default.wd = 0.0 -default.per_batch_size = 20 -default.lr_step = '16000,24000,30000' - -def generate_config(_network, _dataset): - for k, v in network[_network].items(): - config[k] = v - default[k] = v - for k, v in dataset[_dataset].items(): - config[k] = v - default[k] = v - config.network = _network - config.dataset = _dataset - diff --git a/PRNet.mxnet/data.py b/PRNet.mxnet/data.py deleted file mode 100644 index 2ad1bfb..0000000 --- a/PRNet.mxnet/data.py +++ /dev/null @@ -1,164 +0,0 @@ -# pylint: skip-file -import mxnet as mx -import numpy as np -import sys, os -import random -import glob -import math -import scipy.misc -import cv2 -import logging -import sklearn -import datetime -import img_helper -from mxnet.io import DataIter -from mxnet import ndarray as nd -from mxnet import io -from mxnet import recordio -from PIL import Image -from config import config -from skimage import transform as tf - - -class FaceSegIter(DataIter): - def __init__(self, path, batch_size, - per_batch_size = 0, - aug_level = 0, - force_mirror = False, - exf = 1, - args = None): - self.aug_level = aug_level - self.force_mirror = force_mirror - self.exf = exf - self.batch_size = batch_size - self.per_batch_size = per_batch_size - self.image_file_list = [] - self.uv_file_list = [] - for _file in glob.glob(os.path.join(path, '*.jpg')): - self.image_file_list.append(_file) - for img in self.image_file_list: - uv_file = img[0:-3]+"npy" - self.uv_file_list.append(uv_file) - self.seq = range(len(self.image_file_list)) - print('train size', len(self.seq)) - self.cur = 0 - self.reset() - self.data_shape = (3, config.input_img_size, config.input_img_size) - self.num_classes = config.num_classes - self.input_img_size = config.input_img_size - #self.label_classes = self.num_classes - self.output_label_size = config.output_label_size - #if aug_level>0: - # self.output_label_size = config.output_label_size - #else: - # self.output_label_size = self.input_img_size - self.label_shape = (self.num_classes, self.output_label_size, self.output_label_size) - self.provide_data = [('data', (batch_size,) + self.data_shape)] - self.provide_label = [('softmax_label', (batch_size,) + self.label_shape), - ('mask_label', (batch_size,)+ self.label_shape)] - weight_mask = cv2.imread('./uv-data/uv_weight_mask.png') - print('weight_mask', weight_mask.shape) - if weight_mask.shape[0]!=self.output_label_size: - weight_mask = cv2.resize(weight_mask, (self.output_label_size, self.output_label_size) ) - #idx = np.where(weight_mask>0)[0] - #print('weight idx', idx) - weight_mask = weight_mask.astype(np.float32) - weight_mask /= 255.0 - - vis_mask = cv2.imread('./uv-data/uv_face_mask.png') - print('vis_mask', vis_mask.shape) - if vis_mask.shape[0]!=self.output_label_size: - vis_mask = cv2.resize(vis_mask, (self.output_label_size, self.output_label_size) ) - vis_mask = vis_mask.astype(np.float32) - vis_mask /= 255.0 - weight_mask *= vis_mask - print('weight_mask', weight_mask.shape) - weight_mask = weight_mask.transpose( (2,0,1) ) - #WM = np.zeros( (batch_size,)+self.label_shape, dtype=np.float32 ) - #for i in range(batch_size): - # WM[i] = weight_mask - #weight_mask = WM - #weight_mask = weight_mask.reshape( (1, 3, weight_mask.shape[0], weight_mask.shape[1]) ) - weight_mask = weight_mask[np.newaxis,:,:,:] - print('weight_mask', weight_mask.shape) - weight_mask = np.tile(weight_mask, (batch_size,1,1,1)) - print('weight_mask', weight_mask.shape) - self.weight_mask = nd.array(weight_mask) - self.img_num = 0 - self.invalid_num = 0 - self.mode = 1 - self.vis = 0 - self.stats = [0,0] - - def get_data_shape(self): - return self.data_shape - - #def get_label_shape(self): - # return self.label_shape - - def get_shape_dict(self): - D = {} - for (k,v) in self.provide_data: - D[k] = v - for (k,v) in self.provide_label: - D[k] = v - return D - - def get_label_names(self): - D = [] - for (k,v) in self.provide_label: - D.append(k) - return D - - def reset(self): - #print('reset') - self.cur = 0 - if self.aug_level>0: - random.shuffle(self.seq) - - def next_sample(self): - """Helper function for reading in next sample.""" - if self.cur >= len(self.seq): - raise StopIteration - idx = self.seq[self.cur] - self.cur += 1 - uv_path = self.uv_file_list[idx] - image_path = self.image_file_list[idx] - uvmap = np.load(uv_path) - img = cv2.imread(image_path)[:,:,::-1]#to rgb - hlabel = uvmap - #print(hlabel.shape) - #hlabel = np.array(header.label).reshape( (self.output_label_size, self.output_label_size, self.num_classes) ) - hlabel /= self.input_img_size - - return img, hlabel - - - def next(self): - """Returns the next batch of data.""" - #print('next') - batch_size = self.batch_size - batch_data = nd.empty((batch_size,)+self.data_shape) - batch_label = nd.empty((batch_size,)+self.label_shape) - i = 0 - #self.cutoff = random.randint(800,1280) - try: - while i < batch_size: - #print('N', i) - data, label = self.next_sample() - data = nd.array(data) - data = nd.transpose(data, axes=(2, 0, 1)) - label = nd.array(label) - label = nd.transpose(label, axes=(2, 0, 1)) - batch_data[i][:] = data - batch_label[i][:] = label - i += 1 - except StopIteration: - if i1 or not dim_match: - return conv_resnet(data, num_filter, stride, dim_match, name, binarize, dcn, dilate, **kwargs) - conv4 = block35(data, num_filter, name=name+'_block35') - return conv4 - -def conv_cab(data, num_filter, stride, dim_match, name, binarize, dcn, dilate, **kwargs): - if stride[0]>1 or not dim_match: - return conv_hpm(data, num_filter, stride, dim_match, name, binarize, dcn, dilate, **kwargs) - cab = CAB(data, num_filter, 1, 4, workspace, name, dilate, 1) - return cab.get() - -def conv_block(data, num_filter, stride, dim_match, name, binarize, dcn, dilate): - if config.net_block=='resnet': - return conv_resnet(data, num_filter, stride, dim_match, name, binarize, dcn, dilate) - elif config.net_block=='inception': - return conv_inception(data, num_filter, stride, dim_match, name, binarize, dcn, dilate) - elif config.net_block=='hpm': - return conv_hpm(data, num_filter, stride, dim_match, name, binarize, dcn, dilate) - elif config.net_block=='cab': - return conv_cab(data, num_filter, stride, dim_match, name, binarize, dcn, dilate) - elif config.net_block=='prnet': - return conv_prnet(data, num_filter, stride, dim_match, name, binarize, dcn, dilate) - -def hourglass(data, nFilters, nModules, n, workspace, name, binarize, dcn): - s = 2 - _dcn = False - up1 = data - for i in xrange(nModules): - up1 = conv_block(up1, nFilters, (1,1), True, "%s_up1_%d"%(name,i), binarize, _dcn, 1) - low1 = mx.sym.Pooling(data=data, kernel=(s, s), stride=(s,s), pad=(0,0), pool_type='max') - #low1 = ConvFactory(data, nFilters, (4,4), stride=(2,2), pad=(1,1), name=name+'_conv') - #low1 = ConvFactory(data, nFilters, (3,3), stride=(2,2), pad=(1,1), name=name+'_conv') - #low1 = ConvFactory(up1, nFilters, (3,3), stride=(2,2), pad=(1,1), name=name+'_conv') - for i in xrange(nModules): - low1 = conv_block(low1, nFilters, (1,1), True, "%s_low1_%d"%(name,i), binarize, _dcn, 1) - if n>1: - low2 = hourglass(low1, nFilters, nModules, n-1, workspace, "%s_%d"%(name, n-1), binarize, dcn) - else: - low2 = low1 - for i in xrange(nModules): - low2 = conv_block(low2, nFilters, (1,1), True, "%s_low2_%d"%(name,i), binarize, _dcn, 1) #TODO - low3 = low2 - for i in xrange(nModules): - low3 = conv_block(low3, nFilters, (1,1), True, "%s_low3_%d"%(name,i), binarize, _dcn, 1) - up2 = mx.symbol.UpSampling(low3, scale=s, sample_type='nearest', workspace=512, name='%s_upsampling_%s'%(name,n), num_args=1) - #up2 = mx.symbol.UpSampling(low3, scale=s, sample_type='bilinear', num_filter=nFilters, workspace=512, name='%s_upsampling_%s'%(name,n), num_args=1) - #up2 = mx.symbol.Deconvolution(data=low3, num_filter=nFilters, kernel=(s*2,s*2), - # stride=(s, s), pad=(s//2, s//2), - # name='%s_upsampling_%s'%(name,n), - # attr={'lr_mult': '0.1'}) - #return mx.symbol.add_n(up1, up2) - return up2 - - -def prnet_loss(pred, gt_label, mask_label): - loss = pred - gt_label - #loss = mx.symbol.smooth_l1(loss, scalar=3.0) - loss = mx.symbol.abs(loss) - loss = mx.symbol.broadcast_mul(loss, mask_label) - #loss = mx.symbol.mean(loss, axis=0) - #loss = loss*loss - #loss = mx.symbol.mean(loss) - return loss - -def ce_loss(x, y): - #loss = mx.sym.SoftmaxOutput(data = x, label = y, normalization='valid', multi_output=True) - x_max = mx.sym.max(x, axis=[2,3], keepdims=True) - x = mx.sym.broadcast_minus(x, x_max) - body = mx.sym.exp(x) - sums = mx.sym.sum(body, axis=[2,3], keepdims=True) - body = mx.sym.broadcast_div(body, sums) - loss = mx.sym.log(body) - loss = loss*y*-1.0 - #loss = mx.symbol.mean(loss, axis=[1,2,3]) - loss = mx.symbol.mean(loss) - return loss - -def get_symbol(num_classes): - m = config.multiplier - sFilters = max(int(64*m), 16) - mFilters = max(int(128*m), 32) - nFilters = int(256*m) - - nModules = config.net_modules - nStacks = config.net_stacks - binarize = config.net_binarize - input_size = config.input_img_size - label_size = config.output_label_size - use_STA = config.net_sta - N = config.net_n - DCN = config.net_dcn - per_batch_size = config.per_batch_size - print('binarize', binarize) - print('use_STA', use_STA) - print('use_N', N) - print('use_DCN', DCN) - print('per_batch_size', per_batch_size) - #assert(label_size==64 or label_size==32) - #assert(input_size==128 or input_size==256) - D = input_size // label_size - print(input_size, label_size, D) - data = mx.sym.Variable(name='data') - data = data-127.5 - data = data*0.0078125 - gt_label = mx.symbol.Variable(name='softmax_label') - mask_label = mx.symbol.Variable(name='mask_label') - losses = [] - closses = [] - #body = Conv(data=data, num_filter=sFilters, kernel=(3, 3), stride=(1,1), pad=(1, 1), - # no_bias=True, name="conv0", workspace=workspace) - body = Conv(data=data, num_filter=sFilters, kernel=(7,7), stride=(2,2), pad=(3,3), - no_bias=True, name="conv0", workspace=workspace) - #body = Conv(data=data, num_filter=sFilters, kernel=(4,4), stride=(2,2), pad=(1,1), - # no_bias=True, name="conv0", workspace=workspace) - body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn0') - body = Act(data=body, act_type='relu', name='relu0') - - dcn = False - body = conv_block(body, mFilters, (1,1), sFilters==mFilters, 'res0', False, dcn, 1) - body = mx.sym.Pooling(data=body, kernel=(2, 2), stride=(2,2), pad=(0,0), pool_type='max') - - #body = Conv(data=body, num_filter=mFilters, kernel=(4,4), stride=(2,2), pad=(1,1), - # no_bias=True, name="conv1", workspace=workspace) - #body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn1') - #body = Act(data=body, act_type='relu', name='relu1') - - #body = conv_block(body, mFilters, (1,1), True, 'res1', False, dcn, 1) #TODO - body = conv_block(body, nFilters, (1,1), mFilters==nFilters, 'res2', binarize, dcn, 1) #binarize=True? - - heatmap = None - outs = [] - - body = hourglass(body, nFilters, nModules, config.net_n, workspace, 'stack0_hg', binarize, dcn) - for j in xrange(nModules): - body = conv_block(body, nFilters, (1,1), True, 'stack0_unit%d'%(j), binarize, dcn, 1) - _dcn = False - ll = ConvFactory(body, nFilters, (1,1), dcn = _dcn, name='stack0_ll') - _name = 'heatmap' - pred = Conv(data=ll, num_filter=num_classes, kernel=(1, 1), stride=(1,1), pad=(0,0), - name=_name, workspace=workspace) - loss = prnet_loss(pred, gt_label, mask_label) - outs.append(mx.sym.MakeLoss(loss)) - - - pred = mx.symbol.BlockGrad(pred) - #loss = mx.symbol.add_n(*losses) - #loss = mx.symbol.MakeLoss(loss) - #syms = [loss] - outs.append(pred) - sym = mx.symbol.Group( outs ) - return sym - -def init_weights(sym, data_shape_dict): - #print('in hg') - arg_name = sym.list_arguments() - aux_name = sym.list_auxiliary_states() - arg_shape, _, aux_shape = sym.infer_shape(**data_shape_dict) - arg_shape_dict = dict(zip(arg_name, arg_shape)) - aux_shape_dict = dict(zip(aux_name, aux_shape)) - #print(aux_shape) - #print(aux_params) - #print(arg_shape_dict) - arg_params = {} - aux_params = {} - for k,v in arg_shape_dict.iteritems(): - #print(k,v) - if k.endswith('offset_weight') or k.endswith('offset_bias'): - print('initializing',k) - arg_params[k] = mx.nd.zeros(shape = v) - elif k.startswith('fc6_'): - if k.endswith('_weight'): - print('initializing',k) - arg_params[k] = mx.random.normal(0, 0.01, shape=v) - elif k.endswith('_bias'): - print('initializing',k) - arg_params[k] = mx.nd.zeros(shape=v) - elif k.find('upsampling')>=0: - print('initializing upsampling_weight', k) - arg_params[k] = mx.nd.zeros(shape=arg_shape_dict[k]) - init = mx.init.Initializer() - init._init_bilinear(k, arg_params[k]) - return arg_params, aux_params - diff --git a/PRNet.mxnet/train.py b/PRNet.mxnet/train.py deleted file mode 100644 index c4bb954..0000000 --- a/PRNet.mxnet/train.py +++ /dev/null @@ -1,215 +0,0 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import logging -import argparse -from data import FaceSegIter -import mxnet as mx -import mxnet.optimizer as optimizer -import numpy as np -import os -import sys -import math -import random -import cv2 -from config import config, default, generate_config -from optimizer import ONadam -from metric import LossValueMetric, NMEMetric -sys.path.append(os.path.join(os.path.dirname(__file__), 'symbol')) -import sym_heatmap -#import sym_fc -#from symbol import fc - - -args = None -logger = logging.getLogger() -logger.setLevel(logging.INFO) - - -def main(args): - _seed = 727 - random.seed(_seed) - np.random.seed(_seed) - mx.random.seed(_seed) - ctx = [] - cvd = os.environ['CUDA_VISIBLE_DEVICES'].strip() - if len(cvd)>0: - for i in xrange(len(cvd.split(','))): - ctx.append(mx.gpu(i)) - if len(ctx)==0: - ctx = [mx.cpu()] - print('use cpu') - else: - print('gpu num:', len(ctx)) - #ctx = [mx.gpu(0)] - args.ctx_num = len(ctx) - - args.batch_size = args.per_batch_size*args.ctx_num - config.per_batch_size = args.per_batch_size - - - - print('Call with', args, config) - train_iter = FaceSegIter(path = config.dataset_path, - batch_size = args.batch_size, - per_batch_size = args.per_batch_size, - aug_level = 1, - exf = args.exf, - args = args, - ) - - data_shape = train_iter.get_data_shape() - #label_shape = train_iter.get_label_shape() - sym = sym_heatmap.get_symbol(num_classes=config.num_classes) - if len(args.pretrained)==0: - #data_shape_dict = {'data' : (args.per_batch_size,)+data_shape, 'softmax_label' : (args.per_batch_size,)+label_shape} - data_shape_dict = train_iter.get_shape_dict() - arg_params, aux_params = sym_heatmap.init_weights(sym, data_shape_dict) - else: - vec = args.pretrained.split(',') - print('loading', vec) - _, arg_params, aux_params = mx.model.load_checkpoint(vec[0], int(vec[1])) - #sym, arg_params, aux_params = get_symbol(args, arg_params, aux_params) - - model = mx.mod.Module( - context = ctx, - symbol = sym, - label_names = train_iter.get_label_names(), - ) - #lr = 1.0e-3 - #lr = 2.5e-4 - #_rescale_grad = 1.0/args.ctx_num - _rescale_grad = 1.0/args.batch_size - #lr = args.lr - #opt = optimizer.Nadam(learning_rate=args.lr, wd=args.wd, rescale_grad=_rescale_grad, clip_gradient=5.0) - if args.optimizer=='onadam': - opt = ONadam(learning_rate=args.lr, wd=args.wd, rescale_grad=_rescale_grad, clip_gradient=5.0) - elif args.optimizer=='nadam': - opt = optimizer.Nadam(learning_rate=args.lr, rescale_grad=_rescale_grad) - elif args.optimizer=='rmsprop': - opt = optimizer.RMSProp(learning_rate=args.lr, rescale_grad=_rescale_grad) - elif args.optimizer=='adam': - opt = optimizer.Adam(learning_rate=args.lr, rescale_grad=_rescale_grad) - else: - opt = optimizer.SGD(learning_rate=args.lr, momentum=0.9, wd=args.wd, rescale_grad=_rescale_grad) - initializer = mx.init.Xavier(rnd_type='gaussian', factor_type="in", magnitude=2) - _cb = mx.callback.Speedometer(args.batch_size, args.frequent) - _metric = LossValueMetric() - #_metric = NMEMetric() - #_metric2 = AccMetric() - #eval_metrics = [_metric, _metric2] - eval_metrics = [_metric] - lr_steps = [int(x) for x in args.lr_step.split(',')] - print('lr-steps', lr_steps) - global_step = [0] - - def val_test(): - all_layers = sym.get_internals() - vsym = all_layers['heatmap_output'] - vmodel = mx.mod.Module(symbol=vsym, context=ctx, label_names = None) - #model.bind(data_shapes=[('data', (args.batch_size, 3, image_size[0], image_size[1]))], label_shapes=[('softmax_label', (args.batch_size,))]) - vmodel.bind(data_shapes=[('data', (args.batch_size,)+data_shape)]) - arg_params, aux_params = model.get_params() - vmodel.set_params(arg_params, aux_params) - for target in config.val_targets: - _file = os.path.join(config.dataset_path, '%s.rec'%target) - if not os.path.exists(_file): - continue - val_iter = FaceSegIter(path_imgrec = _file, - batch_size = args.batch_size, - #batch_size = 4, - aug_level = 0, - args = args, - ) - _metric = LossValueMetric() - val_metric = mx.metric.create(_metric) - val_metric.reset() - val_iter.reset() - diffs = [] - for i, eval_batch in enumerate(val_iter): - #print(eval_batch.data[0].shape, eval_batch.label[0].shape) - batch_data = mx.io.DataBatch(eval_batch.data) - model.forward(batch_data, is_train=False) - _label = eval_batch.label[0].asnumpy() - _pred = model.get_outputs()[-1].asnumpy() - _diff = np.abs(_pred-_label) - _diff = np.mean(_diff)*config.input_img_size - #print('pred', _pred.shape, _label.shape) - #print('diff', _diff) - diffs.append(_diff) - model.update_metric(val_metric, eval_batch.label) - nme_value = val_metric.get_name_value()[0][1] - print('[%d][%s]LOSS: %f'%(global_step[0], target, nme_value)) - print('avg diff', np.mean(diffs)) - - def _batch_callback(param): - _cb(param) - global_step[0]+=1 - mbatch = global_step[0] - for _lr in lr_steps: - if mbatch==_lr: - if args.optimizer=='sgd': - opt.lr *= 0.1 - else: - opt.lr *= 0.5 - print('lr change to', opt.lr) - break - if mbatch%1000==0: - print('lr-batch-epoch:',opt.lr,param.nbatch,param.epoch) - if mbatch>0 and mbatch%args.verbose==0: - val_test() - if args.ckpt==1: - msave = mbatch//args.verbose - print('saving', msave) - arg, aux = model.get_params() - mx.model.save_checkpoint(args.prefix, msave, model.symbol, arg, aux) - if mbatch==lr_steps[-1]: - if args.ckpt==2: - #msave = mbatch//args.verbose - msave = 1 - print('saving', msave) - arg, aux = model.get_params() - mx.model.save_checkpoint(args.prefix, msave, model.symbol, arg, aux) - sys.exit(0) - - train_iter = mx.io.PrefetchingIter(train_iter) - - model.fit(train_iter, - begin_epoch = 0, - num_epoch = 9999, - #eval_data = val_iter, - eval_data = None, - eval_metric = eval_metrics, - kvstore = 'device', - optimizer = opt, - initializer = initializer, - arg_params = arg_params, - aux_params = aux_params, - allow_missing = True, - batch_end_callback = _batch_callback, - epoch_end_callback = None, - ) - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Train face alignment') - # general - parser.add_argument('--network', help='network name', default=default.network, type=str) - parser.add_argument('--dataset', help='dataset name', default=default.dataset, type=str) - args, rest = parser.parse_known_args() - generate_config(args.network, args.dataset) - parser.add_argument('--prefix', default=default.prefix, help='directory to save model.') - parser.add_argument('--pretrained', default=default.pretrained, help='') - parser.add_argument('--optimizer', default='nadam', help='') - parser.add_argument('--lr', type=float, default=default.lr, help='') - parser.add_argument('--wd', type=float, default=default.wd, help='') - parser.add_argument('--per-batch-size', type=int, default=default.per_batch_size, help='') - parser.add_argument('--lr-step', help='learning rate steps (in epoch)', default=default.lr_step, type=str) - parser.add_argument('--ckpt', type=int, default=1, help='') - parser.add_argument('--norm', type=int, default=0, help='') - parser.add_argument('--exf', type=int, default=1, help='') - parser.add_argument('--frequent', type=int, default=default.frequent, help='') - parser.add_argument('--verbose', type=int, default=default.verbose, help='') - args = parser.parse_args() - main(args) - diff --git a/README.md b/README.md index f8963d6..035b25e 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ Please click the image to watch the Youtube video. For Bilibili users, click [he **`2020-10-13`**: A new training method and one large training set(360K IDs) were released [here](https://github.com/deepinsight/insightface/tree/master/recognition/partial_fc) by DeepGlint. -**`2020-10-09`**: We opened a large scale recognition test benchmark [IFRT](https://github.com/deepinsight/insightface/tree/master/IFRT) +**`2020-10-09`**: We opened a large scale recognition test benchmark [IFRT](https://github.com/deepinsight/insightface/tree/master/challenges/IFRT) **`2020-08-01`**: We released lightweight facial landmark models with fast coordinate regression(106 points). See detail [here](https://github.com/deepinsight/insightface/tree/master/alignment/coordinateReg). @@ -34,15 +34,15 @@ Please click the image to watch the Youtube video. For Bilibili users, click [he **`2020.02.21`**: Instant discussion group created on QQ with group-id: 711302608. For English developers, see install tutorial [here](https://github.com/deepinsight/insightface/issues/1069). -**`2020.02.16`**: RetinaFace now can detect faces with mask, for anti-CoVID19, see detail [here](https://github.com/deepinsight/insightface/tree/master/RetinaFaceAntiCov) +**`2020.02.16`**: RetinaFace now can detect faces with mask, for anti-CoVID19, see detail [here](https://github.com/deepinsight/insightface/tree/master/detection/RetinaFaceAntiCov) **`2019.08.10`**: We achieved 2nd place at [WIDER Face Detection Challenge 2019](http://wider-challenge.org/2019.html). **`2019.05.30`**: [Presentation at cvmart](https://pan.baidu.com/s/1v9fFHBJ8Q9Kl9Z6GwhbY6A) -**`2019.04.30`**: Our Face detector ([RetinaFace](https://github.com/deepinsight/insightface/tree/master/RetinaFace)) obtains state-of-the-art results on [the WiderFace dataset](http://shuoyang1213.me/WIDERFACE/WiderFace_Results.html). +**`2019.04.30`**: Our Face detector ([RetinaFace](https://github.com/deepinsight/insightface/tree/master/detection/RetinaFace)) obtains state-of-the-art results on [the WiderFace dataset](http://shuoyang1213.me/WIDERFACE/WiderFace_Results.html). -**`2019.04.14`**: We will launch a [Light-weight Face Recognition challenge/workshop](https://github.com/deepinsight/insightface/tree/master/iccv19-challenge) on ICCV 2019. +**`2019.04.14`**: We will launch a [Light-weight Face Recognition challenge/workshop](https://github.com/deepinsight/insightface/tree/master/challenges/iccv19-lfr) on ICCV 2019. **`2019.04.04`**: Arcface achieved state-of-the-art performance (7/109) on the NIST Face Recognition Vendor Test (FRVT) (1:1 verification) [report](https://www.nist.gov/sites/default/files/documents/2019/04/04/frvt_report_2019_04_04.pdf) (name: Imperial-000 and Imperial-001). Our solution is based on [MS1MV2+DeepGlintAsian, ResNet100, ArcFace loss]. diff --git a/RetinaFace/rcnn/PY_OP/cascade_refine.py b/RetinaFace/rcnn/PY_OP/cascade_refine.py deleted file mode 100644 index fc93ee5..0000000 --- a/RetinaFace/rcnn/PY_OP/cascade_refine.py +++ /dev/null @@ -1,453 +0,0 @@ - -from __future__ import print_function -import sys -import mxnet as mx -import numpy as np -import datetime -from distutils.util import strtobool -from ..config import config, generate_config -from ..processing.generate_anchor import generate_anchors, anchors_plane -from ..processing.bbox_transform import bbox_overlaps, bbox_transform, landmark_transform - -STAT = {0:0} -STEP = 28800 - -class CascadeRefineOperator(mx.operator.CustomOp): - def __init__(self, stride=0, network='', dataset='', prefix=''): - super(CascadeRefineOperator, self).__init__() - self.stride = int(stride) - self.prefix = prefix - generate_config(network, dataset) - self.mode = config.TRAIN.OHEM_MODE #0 for random 10:245, 1 for 10:246, 2 for 10:30, mode 1 for default - stride = self.stride - sstride = str(stride) - base_size = config.RPN_ANCHOR_CFG[sstride]['BASE_SIZE'] - allowed_border = config.RPN_ANCHOR_CFG[sstride]['ALLOWED_BORDER'] - ratios = config.RPN_ANCHOR_CFG[sstride]['RATIOS'] - scales = config.RPN_ANCHOR_CFG[sstride]['SCALES'] - base_anchors = generate_anchors(base_size=base_size, ratios=list(ratios), scales=np.array(scales, dtype=np.float32), stride = stride, dense_anchor = config.DENSE_ANCHOR) - num_anchors = base_anchors.shape[0] - feat_height, feat_width = config.SCALES[0][0]//self.stride, config.SCALES[0][0]//self.stride - feat_stride = self.stride - - A = num_anchors - K = feat_height * feat_width - self.A = A - - all_anchors = anchors_plane(feat_height, feat_width, feat_stride, base_anchors) - all_anchors = all_anchors.reshape((K * A, 4)) - self.ori_anchors = all_anchors - self.nbatch = 0 - global STAT - for k in config.RPN_FEAT_STRIDE: - STAT[k] = [0,0,0] - - def apply_bbox_pred(self, bbox_pred, ind = None): - box_deltas = bbox_pred - box_deltas[:, 0::4] = box_deltas[:,0::4] * config.TRAIN.BBOX_STDS[0] - box_deltas[:, 1::4] = box_deltas[:,1::4] * config.TRAIN.BBOX_STDS[1] - box_deltas[:, 2::4] = box_deltas[:,2::4] * config.TRAIN.BBOX_STDS[2] - box_deltas[:, 3::4] = box_deltas[:,3::4] * config.TRAIN.BBOX_STDS[3] - if ind is None: - boxes = self.ori_anchors - else: - boxes = self.ori_anchors[ind] - #print('in apply',self.stride, box_deltas.shape, boxes.shape) - - widths = boxes[:, 2] - boxes[:, 0] + 1.0 - heights = boxes[:, 3] - boxes[:, 1] + 1.0 - ctr_x = boxes[:, 0] + 0.5 * (widths - 1.0) - ctr_y = boxes[:, 1] + 0.5 * (heights - 1.0) - - dx = box_deltas[:, 0:1] - dy = box_deltas[:, 1:2] - dw = box_deltas[:, 2:3] - dh = box_deltas[:, 3:4] - - pred_ctr_x = dx * widths[:, np.newaxis] + ctr_x[:, np.newaxis] - pred_ctr_y = dy * heights[:, np.newaxis] + ctr_y[:, np.newaxis] - pred_w = np.exp(dw) * widths[:, np.newaxis] - pred_h = np.exp(dh) * heights[:, np.newaxis] - - pred_boxes = np.zeros(box_deltas.shape) - # x1 - pred_boxes[:, 0:1] = pred_ctr_x - 0.5 * (pred_w - 1.0) - # y1 - pred_boxes[:, 1:2] = pred_ctr_y - 0.5 * (pred_h - 1.0) - # x2 - pred_boxes[:, 2:3] = pred_ctr_x + 0.5 * (pred_w - 1.0) - # y2 - pred_boxes[:, 3:4] = pred_ctr_y + 0.5 * (pred_h - 1.0) - return pred_boxes - - def assign_anchor_fpn(self, gt_label, anchors, landmark=False, prefix='face'): - IOU = config.TRAIN.CASCADE_OVERLAP - - gt_boxes = gt_label['gt_boxes'] - #_label = gt_label['gt_label'] - # clean up boxes - #nonneg = np.where(_label[:] != -1)[0] - #gt_boxes = gt_boxes[nonneg] - if landmark: - gt_landmarks = gt_label['gt_landmarks'] - #gt_landmarks = gt_landmarks[nonneg] - assert gt_boxes.shape[0]==gt_landmarks.shape[0] - #scales = np.array(scales, dtype=np.float32) - feat_strides = config.RPN_FEAT_STRIDE - bbox_pred_len = 4 - landmark_pred_len = 10 - num_anchors = anchors.shape[0] - A = self.A - total_anchors = num_anchors - feat_height, feat_width = config.SCALES[0][0]//self.stride, config.SCALES[0][0]//self.stride - - - #print('total_anchors', anchors.shape[0], len(inds_inside), file=sys.stderr) - - # label: 1 is positive, 0 is negative, -1 is dont care - labels = np.empty((num_anchors,), dtype=np.float32) - labels.fill(-1) - #print('BB', anchors.shape, len(inds_inside)) - #print('gt_boxes', gt_boxes.shape, file=sys.stderr) - #tb = datetime.datetime.now() - #self._times[0] += (tb-ta).total_seconds() - #ta = datetime.datetime.now() - - if gt_boxes.size > 0: - # overlap between the anchors and the gt boxes - # overlaps (ex, gt) - overlaps = bbox_overlaps(anchors.astype(np.float), gt_boxes.astype(np.float)) - argmax_overlaps = overlaps.argmax(axis=1) - #print('AAA', argmax_overlaps.shape) - max_overlaps = overlaps[np.arange(num_anchors), argmax_overlaps] - gt_argmax_overlaps = overlaps.argmax(axis=0) - gt_max_overlaps = overlaps[gt_argmax_overlaps, np.arange(overlaps.shape[1])] - gt_argmax_overlaps = np.where(overlaps == gt_max_overlaps)[0] - - if not config.TRAIN.RPN_CLOBBER_POSITIVES: - # assign bg labels first so that positive labels can clobber them - labels[max_overlaps < IOU[0]] = 0 - - # fg label: for each gt, anchor with highest overlap - if config.TRAIN.RPN_FORCE_POSITIVE: - labels[gt_argmax_overlaps] = 1 - - # fg label: above threshold IoU - labels[max_overlaps >= IOU[1]] = 1 - - if config.TRAIN.RPN_CLOBBER_POSITIVES: - # assign bg labels last so that negative labels can clobber positives - labels[max_overlaps < IOU[0]] = 0 - else: - labels[:] = 0 - fg_inds = np.where(labels == 1)[0] - #print('fg count', len(fg_inds)) - - # subsample positive labels if we have too many - if config.TRAIN.RPN_ENABLE_OHEM==0: - fg_inds = np.where(labels == 1)[0] - num_fg = int(config.TRAIN.RPN_FG_FRACTION * config.TRAIN.RPN_BATCH_SIZE) - if len(fg_inds) > num_fg: - disable_inds = npr.choice(fg_inds, size=(len(fg_inds) - num_fg), replace=False) - if DEBUG: - disable_inds = fg_inds[:(len(fg_inds) - num_fg)] - labels[disable_inds] = -1 - - # subsample negative labels if we have too many - num_bg = config.TRAIN.RPN_BATCH_SIZE - np.sum(labels == 1) - bg_inds = np.where(labels == 0)[0] - if len(bg_inds) > num_bg: - disable_inds = npr.choice(bg_inds, size=(len(bg_inds) - num_bg), replace=False) - if DEBUG: - disable_inds = bg_inds[:(len(bg_inds) - num_bg)] - labels[disable_inds] = -1 - - #fg_inds = np.where(labels == 1)[0] - #num_fg = len(fg_inds) - #num_bg = num_fg*int(1.0/config.TRAIN.RPN_FG_FRACTION-1) - - #bg_inds = np.where(labels == 0)[0] - #if len(bg_inds) > num_bg: - # disable_inds = npr.choice(bg_inds, size=(len(bg_inds) - num_bg), replace=False) - # if DEBUG: - # disable_inds = bg_inds[:(len(bg_inds) - num_bg)] - # labels[disable_inds] = -1 - else: - fg_inds = np.where(labels == 1)[0] - num_fg = len(fg_inds) - bg_inds = np.where(labels == 0)[0] - num_bg = len(bg_inds) - - #print('anchor stat', num_fg, num_bg) - - - bbox_targets = np.zeros((num_anchors, bbox_pred_len), dtype=np.float32) - if gt_boxes.size > 0: - #print('GT', gt_boxes.shape, gt_boxes[argmax_overlaps, :4].shape) - bbox_targets[:,:] = bbox_transform(anchors, gt_boxes[argmax_overlaps, :]) - #bbox_targets[:,4] = gt_blur - #tb = datetime.datetime.now() - #self._times[1] += (tb-ta).total_seconds() - #ta = datetime.datetime.now() - - bbox_weights = np.zeros((num_anchors, bbox_pred_len), dtype=np.float32) - #bbox_weights[labels == 1, :] = np.array(config.TRAIN.RPN_BBOX_WEIGHTS) - bbox_weights[labels == 1, 0:4] = 1.0 - if bbox_pred_len>4: - bbox_weights[labels == 1, 4:bbox_pred_len] = 0.1 - - if landmark: - landmark_targets = np.zeros((num_anchors, landmark_pred_len), dtype=np.float32) - landmark_weights = np.zeros((num_anchors, landmark_pred_len), dtype=np.float32) - #landmark_weights[labels == 1, :] = np.array(config.TRAIN.RPN_LANDMARK_WEIGHTS) - if landmark_pred_len==10: - landmark_weights[labels == 1, :] = 1.0 - elif landmark_pred_len==15: - v = [1.0, 1.0, 0.1] * 5 - assert len(v)==15 - landmark_weights[labels == 1, :] = np.array(v) - else: - assert False - #TODO here - if gt_landmarks.size > 0: - #print('AAA',argmax_overlaps) - a_landmarks = gt_landmarks[argmax_overlaps,:,:] - landmark_targets[:] = landmark_transform(anchors, a_landmarks) - invalid = np.where(a_landmarks[:,0,2]<0.0)[0] - #assert len(invalid)==0 - #landmark_weights[invalid, :] = np.array(config.TRAIN.RPN_INVALID_LANDMARK_WEIGHTS) - landmark_weights[invalid, :] = 0.0 - #tb = datetime.datetime.now() - #self._times[2] += (tb-ta).total_seconds() - #ta = datetime.datetime.now() - bbox_targets[:, 0::4] = bbox_targets[:,0::4] / config.TRAIN.BBOX_STDS[0] - bbox_targets[:, 1::4] = bbox_targets[:,1::4] / config.TRAIN.BBOX_STDS[1] - bbox_targets[:, 2::4] = bbox_targets[:,2::4] / config.TRAIN.BBOX_STDS[2] - bbox_targets[:, 3::4] = bbox_targets[:,3::4] / config.TRAIN.BBOX_STDS[3] - - #print('CC', anchors.shape, len(inds_inside)) - label = {} - _label = labels.reshape((1, feat_height, feat_width, A)).transpose(0, 3, 1, 2) - _label = _label.reshape((1, A * feat_height * feat_width)) - bbox_target = bbox_targets.reshape((1, feat_height*feat_width, A * bbox_pred_len)).transpose(0, 2, 1) - bbox_weight = bbox_weights.reshape((1, feat_height*feat_width, A * bbox_pred_len)).transpose((0, 2, 1)) - label['%s_label'%prefix] = _label[0] - label['%s_bbox_target'%prefix] = bbox_target[0] - label['%s_bbox_weight'%prefix] = bbox_weight[0] - if landmark: - landmark_target = landmark_target.reshape((1, feat_height*feat_width, A * landmark_pred_len)).transpose(0, 2, 1) - landmark_target /= config.TRAIN.LANDMARK_STD - landmark_weight = landmark_weight.reshape((1, feat_height*feat_width, A * landmark_pred_len)).transpose((0, 2, 1)) - label['%s_landmark_target'%prefix] = landmark_target[0] - label['%s_landmark_weight'%prefix] = landmark_weight[0] - - return label - - def forward(self, is_train, req, in_data, out_data, aux): - self.nbatch+=1 - ta = datetime.datetime.now() - global STAT - A = config.NUM_ANCHORS - - cls_label_t0 = in_data[0].asnumpy() #BS, AHW - cls_score_t0 = in_data[1].asnumpy() #BS, C, AHW - cls_score = in_data[2].asnumpy() #BS, C, AHW - #labels_raw = in_data[1].asnumpy() #BS, ANCHORS - bbox_pred_t0 = in_data[3].asnumpy() #BS, AC, HW - bbox_target_t0 = in_data[4].asnumpy() #BS, AC, HW - cls_label_raw = in_data[5].asnumpy() #BS, AHW - gt_boxes = in_data[6].asnumpy() #BS, N, C=4+1 - #imgs = in_data[7].asnumpy().astype(np.uint8) - - batch_size = cls_score.shape[0] - num_anchors = cls_score.shape[2] - #print('in cas', cls_score.shape, bbox_target.shape) - - labels_out = np.zeros(shape=(batch_size, num_anchors), dtype=np.float32) - bbox_target_out = np.zeros(shape=bbox_target_t0.shape, dtype=np.float32) - anchor_weight = np.zeros( (batch_size, num_anchors,1), dtype=np.float32 ) - valid_count = np.zeros( (batch_size,1), dtype=np.float32 ) - - bbox_pred_t0 = bbox_pred_t0.transpose( (0,2,1) ) - bbox_pred_t0 = bbox_pred_t0.reshape( (bbox_pred_t0.shape[0], -1, 4) ) #BS, H*W*A, C - bbox_target_t0 = bbox_target_t0.transpose( (0,2,1) ) - bbox_target_t0 = bbox_target_t0.reshape( (bbox_target_t0.shape[0], -1, 4) ) - - #print('anchor_weight', anchor_weight.shape) - - #assert labels.shape[0]==1 - #assert cls_score.shape[0]==1 - #assert bbox_weight.shape[0]==1 - #print('shape', cls_score.shape, labels.shape, file=sys.stderr) - #print('bbox_weight 0', bbox_weight.shape, file=sys.stderr) - #bbox_weight = np.zeros( (labels_raw.shape[0], labels_raw.shape[1], 4), dtype=np.float32) - _stat = [0,0,0] - SEL_TOPK = config.TRAIN.RPN_BATCH_SIZE - FAST = False - for ibatch in range(batch_size): - #bgr = imgs[ibatch].transpose( (1,2,0) )[:,:,::-1] - - if not FAST: - _gt_boxes = gt_boxes[ibatch] #N, 4+1 - _gtind = len(np.where(_gt_boxes[:,4]>=0)[0]) - #print('gt num', _gtind) - _gt_boxes = _gt_boxes[0:_gtind,:] - - #anchors_t1 = self.ori_anchors.copy() - #_cls_label_raw = cls_label_raw[ibatch] #AHW - #_cls_label_raw = _cls_label_raw.reshape( (A, -1) ).transpose( (1,0) ).reshape( (-1,) ) #HWA - #fg_ind_raw = np.where(_cls_label_raw>0)[0] - #_bbox_target_t0 = bbox_target_t0[ibatch][fg_ind_raw] - #_bbox_pred_t0 = bbox_pred_t0[ibatch][fg_ind_raw] - #anchors_t1_pos = self.apply_bbox_pred(_bbox_pred_t0, ind=fg_ind_raw) - #anchors_t1[fg_ind_raw,:] = anchors_t1_pos - - anchors_t1 = self.apply_bbox_pred(bbox_pred_t0[ibatch]) - assert anchors_t1.shape[0]==self.ori_anchors.shape[0] - - #for i in range(_gt_boxes.shape[0]): - # box = _gt_boxes[i].astype(np.int) - # print('%d: gt%d'%(self.nbatch, i), box) - # #color = (0,0,255) - # #cv2.rectangle(img, (box[0], box[1]), (box[2], box[3]), color, 2) - #for i in range(anchors_t1.shape[0]): - # box1 = self.ori_anchors[i].astype(np.int) - # box2 = anchors_t1[i].astype(np.int) - # print('%d %d: anchorscompare %d'%(self.nbatch, self.stride, i), box1, box2) - #color = (255,255,0) - #cv2.rectangle(img, (box[0], box[1]), (box[2], box[3]), color, 2) - #filename = "./debug/%d_%d_%d.jpg"%(self.nbatch, ibatch, stride) - #cv2.imwrite(filename, img) - #print(filename) - #gt_label = {'gt_boxes': gt_anchors, 'gt_label' : labels_raw[ibatch]} - gt_label = {'gt_boxes': _gt_boxes} - new_label_dict = self.assign_anchor_fpn(gt_label, anchors_t1, False, prefix=self.prefix) - labels = new_label_dict['%s_label'%self.prefix] #AHW - new_bbox_target = new_label_dict['%s_bbox_target'%self.prefix] #AC,HW - #print('assign ret', labels.shape, new_bbox_target.shape) - _anchor_weight = np.zeros( (num_anchors,1), dtype=np.float32) - fg_score = cls_score[ibatch,1,:] - cls_score[ibatch,0,:] - fg_inds = np.where(labels>0)[0] - num_fg = int(config.TRAIN.RPN_FG_FRACTION * config.TRAIN.RPN_BATCH_SIZE) - origin_num_fg = len(fg_inds) - #continue - #print('cas fg', len(fg_inds), num_fg, file=sys.stderr) - if len(fg_inds) > num_fg: - if self.mode==0: - disable_inds = np.random.choice(fg_inds, size=(len(fg_inds) - num_fg), replace=False) - labels[disable_inds] = -1 - else: - pos_ohem_scores = fg_score[fg_inds] - order_pos_ohem_scores = pos_ohem_scores.ravel().argsort() - sampled_inds = fg_inds[order_pos_ohem_scores[:num_fg]] - labels[fg_inds] = -1 - labels[sampled_inds] = 1 - - n_fg = np.sum(labels>0) - fg_inds = np.where(labels>0)[0] - num_bg = config.TRAIN.RPN_BATCH_SIZE - n_fg - if self.mode==2: - num_bg = max(48, n_fg*int(1.0/config.TRAIN.RPN_FG_FRACTION-1)) - - bg_inds = np.where(labels == 0)[0] - origin_num_bg = len(bg_inds) - if num_bg==0: - labels[bg_inds] = -1 - elif len(bg_inds) > num_bg: - # sort ohem scores - - if self.mode==0: - disable_inds = np.random.choice(bg_inds, size=(len(bg_inds) - num_bg), replace=False) - labels[disable_inds] = -1 - else: - neg_ohem_scores = fg_score[bg_inds] - order_neg_ohem_scores = neg_ohem_scores.ravel().argsort()[::-1] - sampled_inds = bg_inds[order_neg_ohem_scores[:num_bg]] - #print('sampled_inds_bg', sampled_inds, file=sys.stderr) - labels[bg_inds] = -1 - labels[sampled_inds] = 0 - - if n_fg>0: - order0_labels = labels.reshape( (1, A, -1) ).transpose( (0, 2, 1) ).reshape( (-1,) ) - bbox_fg_inds = np.where(order0_labels>0)[0] - #print('bbox_fg_inds, order0 ', bbox_fg_inds, file=sys.stderr) - _anchor_weight[bbox_fg_inds,:] = 1.0 - anchor_weight[ibatch] = _anchor_weight - valid_count[ibatch][0] = n_fg - labels_out[ibatch] = labels - #print('labels_out', self.stride, ibatch, labels) - bbox_target_out[ibatch] = new_bbox_target - #print('cascade stat', self.stride, ibatch, len(labels), len(np.where(labels==1)[0]), len(np.where(labels==0)[0])) - else: #FAST MODE - fg_score_t0 = cls_score_t0[ibatch,1,:] - cls_score_t0[ibatch,0,:] - sort_idx_t0 = np.argsort(fg_score_t0.flatten())[::-1][0:SEL_TOPK] - _bbox_pred_t0 = bbox_pred_t0[ibatch][sort_idx_t0] - _bbox_target_t0 = bbox_target_t0[ibatch][sort_idx_t0] - #print('SEL fg score:', fg_score_t0[sort_idx[-1]], fg_score_t0[sort_idx[0]]) - anchors_t0 = self.apply_bbox_pred(_bbox_pred_t0) - gt_anchors = self.apply_bbox_pred(_bbox_target_t0) - #gt_label = {'gt_boxes': gt_anchors, 'gt_label' : labels_raw[ibatch]} - gt_label = {'gt_boxes': gt_anchors} - new_label_dict = self.assign_anchor_fpn(gt_label, anchors_t0, False, prefix=self.prefix) - labels = new_label_dict['%s_label'%self.prefix] - new_bbox_target = new_label_dict['%s_bbox_target'%self.prefix] - #print('assign ret', labels.shape, new_bbox_target.shape) - _anchor_weight = np.zeros( (num_anchors,1), dtype=np.float32) - fg_score = cls_score[ibatch,1,:] - cls_score[ibatch,0,:] - fg_inds = np.where(labels>0)[0] - _labels = np.empty(shape=labels.shape, dtype=np.float32) - _labels.fill(-1) - _labels[sort_idx_idx] = labels - - anchor_weight[ibatch] = _anchor_weight - valid_count[ibatch][0] = len(fg_inds) - labels_out[ibatch] = _labels - #print('labels_out', self.stride, ibatch, labels) - bbox_target_out[ibatch] = new_bbox_target - - #print('cascade pos stat', self.stride, batch_size, np.sum(valid_count)) - for ind, val in enumerate([labels_out, bbox_target_out, anchor_weight, valid_count]): - val = mx.nd.array(val) - self.assign(out_data[ind], req[ind], val) - tb = datetime.datetime.now() - #print('cascade forward cost', self.stride, (tb-ta).total_seconds()) - - def backward(self, req, out_grad, in_data, out_data, in_grad, aux): - for i in range(len(in_grad)): - self.assign(in_grad[i], req[i], 0) - - -@mx.operator.register('cascade_refine') -class CascadeRefineProp(mx.operator.CustomOpProp): - def __init__(self, stride=0, network='', dataset='', prefix=''): - super(CascadeRefineProp, self).__init__(need_top_grad=False) - self.stride = stride - self.network=network - self.dataset=dataset - self.prefix = prefix - - def list_arguments(self): - #return ['cls_label_t0', 'cls_pred_t0', 'cls_pred', 'bbox_pred_t0', 'bbox_label_t0', 'cls_label_raw', 'cas_gt_boxes', 'cas_img'] - return ['cls_label_t0', 'cls_pred_t0', 'cls_pred', 'bbox_pred_t0', 'bbox_label_t0', 'cls_label_raw', 'cas_gt_boxes'] - - def list_outputs(self): - return ['cls_label_out', 'bbox_label_out', 'anchor_weight_out', 'pos_count_out'] - - def infer_shape(self, in_shape): - cls_pred_shape = in_shape[1] - bs = cls_pred_shape[0] - num_anchors = cls_pred_shape[2] - #print('in_rpn_ohem', in_shape[0], in_shape[1], in_shape[2], file=sys.stderr) - #print('in_rpn_ohem', labels_shape, anchor_weight_shape) - cls_label_shape = [bs, num_anchors] - - return in_shape, \ - [cls_label_shape, in_shape[4], [bs,num_anchors,1], [bs,1]] - - def create_operator(self, ctx, shapes, dtypes): - return CascadeRefineOperator(self.stride, self.network, self.dataset, self.prefix) - - def declare_backward_dependency(self, out_grad, in_data, out_data): - return [] - - diff --git a/RetinaFace/rcnn/PY_OP/rpn_fpn_ohem3.py b/RetinaFace/rcnn/PY_OP/rpn_fpn_ohem3.py deleted file mode 100644 index 330db57..0000000 --- a/RetinaFace/rcnn/PY_OP/rpn_fpn_ohem3.py +++ /dev/null @@ -1,167 +0,0 @@ - -from __future__ import print_function -import sys -import mxnet as mx -import numpy as np -from distutils.util import strtobool -from ..config import config, generate_config - - -STAT = {0:0} -STEP = 28800 - -class RPNFPNOHEM3Operator(mx.operator.CustomOp): - def __init__(self, stride=0, network='', dataset='', prefix=''): - super(RPNFPNOHEM3Operator, self).__init__() - self.stride = int(stride) - self.prefix = prefix - generate_config(network, dataset) - self.mode = config.TRAIN.OHEM_MODE #0 for random 10:245, 1 for 10:246, 2 for 10:30, mode 1 for default - global STAT - for k in config.RPN_FEAT_STRIDE: - STAT[k] = [0,0,0] - - def forward(self, is_train, req, in_data, out_data, aux): - global STAT - - cls_score = in_data[0].asnumpy() #BS, 2, ANCHORS - labels_raw = in_data[1].asnumpy() # BS, ANCHORS - - A = config.NUM_ANCHORS - anchor_weight = np.zeros( (labels_raw.shape[0], labels_raw.shape[1],1), dtype=np.float32 ) - valid_count = np.zeros( (labels_raw.shape[0],1), dtype=np.float32 ) - #print('anchor_weight', anchor_weight.shape) - - #assert labels.shape[0]==1 - #assert cls_score.shape[0]==1 - #assert bbox_weight.shape[0]==1 - #print('shape', cls_score.shape, labels.shape, file=sys.stderr) - #print('bbox_weight 0', bbox_weight.shape, file=sys.stderr) - #bbox_weight = np.zeros( (labels_raw.shape[0], labels_raw.shape[1], 4), dtype=np.float32) - _stat = [0,0,0] - for ibatch in range(labels_raw.shape[0]): - _anchor_weight = np.zeros( (labels_raw.shape[1],1), dtype=np.float32) - labels = labels_raw[ibatch] - fg_score = cls_score[ibatch,1,:] - cls_score[ibatch,0,:] - - - - fg_inds = np.where(labels>0)[0] - num_fg = int(config.TRAIN.RPN_FG_FRACTION * config.TRAIN.RPN_BATCH_SIZE) - origin_num_fg = len(fg_inds) - #print(len(fg_inds), num_fg, file=sys.stderr) - if len(fg_inds) > num_fg: - if self.mode==0: - disable_inds = np.random.choice(fg_inds, size=(len(fg_inds) - num_fg), replace=False) - labels[disable_inds] = -1 - else: - pos_ohem_scores = fg_score[fg_inds] - order_pos_ohem_scores = pos_ohem_scores.ravel().argsort() - sampled_inds = fg_inds[order_pos_ohem_scores[:num_fg]] - labels[fg_inds] = -1 - labels[sampled_inds] = 1 - - n_fg = np.sum(labels>0) - fg_inds = np.where(labels>0)[0] - num_bg = config.TRAIN.RPN_BATCH_SIZE - n_fg - if self.mode==2: - num_bg = max(48, n_fg*int(1.0/config.TRAIN.RPN_FG_FRACTION-1)) - - bg_inds = np.where(labels == 0)[0] - origin_num_bg = len(bg_inds) - if num_bg==0: - labels[bg_inds] = -1 - elif len(bg_inds) > num_bg: - # sort ohem scores - - if self.mode==0: - disable_inds = np.random.choice(bg_inds, size=(len(bg_inds) - num_bg), replace=False) - labels[disable_inds] = -1 - else: - neg_ohem_scores = fg_score[bg_inds] - order_neg_ohem_scores = neg_ohem_scores.ravel().argsort()[::-1] - sampled_inds = bg_inds[order_neg_ohem_scores[:num_bg]] - #print('sampled_inds_bg', sampled_inds, file=sys.stderr) - labels[bg_inds] = -1 - labels[sampled_inds] = 0 - - if n_fg>0: - order0_labels = labels.reshape( (1, A, -1) ).transpose( (0, 2, 1) ).reshape( (-1,) ) - bbox_fg_inds = np.where(order0_labels>0)[0] - #print('bbox_fg_inds, order0 ', bbox_fg_inds, file=sys.stderr) - _anchor_weight[bbox_fg_inds,:] = 1.0 - anchor_weight[ibatch] = _anchor_weight - valid_count[ibatch][0] = n_fg - - #if self.prefix=='face': - # #print('fg-bg', self.stride, n_fg, num_bg) - # STAT[0]+=1 - # STAT[self.stride][0] += config.TRAIN.RPN_BATCH_SIZE - # STAT[self.stride][1] += n_fg - # STAT[self.stride][2] += np.sum(fg_score[fg_inds]>=0) - # #_stat[0] += config.TRAIN.RPN_BATCH_SIZE - # #_stat[1] += n_fg - # #_stat[2] += np.sum(fg_score[fg_inds]>=0) - # #print('stride num_fg', self.stride, n_fg, file=sys.stderr) - # #ACC[self.stride] += np.sum(fg_score[fg_inds]>=0) - # #x = float(labels_raw.shape[0]*len(config.RPN_FEAT_STRIDE)) - # x = 1.0 - # if STAT[0]%STEP==0: - # _str = ['STAT'] - # STAT[0] = 0 - # for k in config.RPN_FEAT_STRIDE: - # acc = float(STAT[k][2])/STAT[k][1] - # acc0 = float(STAT[k][1])/STAT[k][0] - # #_str.append("%d: all-fg(%d, %d, %.4f), fg-fgcorrect(%d, %d, %.4f)"%(k,STAT[k][0], STAT[k][1], acc0, STAT[k][1], STAT[k][2], acc)) - # _str.append("%d: (%d, %d, %.4f)"%(k, STAT[k][1], STAT[k][2], acc)) - # STAT[k] = [0,0,0] - # _str = ' | '.join(_str) - # print(_str, file=sys.stderr) - #if self.stride==4 and num_fg>0: - # print('_stat_', self.stride, num_fg, num_bg, file=sys.stderr) - - #labels_ohem = mx.nd.array(labels_raw) - #anchor_weight = mx.nd.array(anchor_weight) - #print('valid_count', self.stride, np.sum(valid_count)) - #print('_stat', _stat, valid_count) - - for ind, val in enumerate([labels_raw, anchor_weight, valid_count]): - val = mx.nd.array(val) - self.assign(out_data[ind], req[ind], val) - - def backward(self, req, out_grad, in_data, out_data, in_grad, aux): - for i in range(len(in_grad)): - self.assign(in_grad[i], req[i], 0) - - -@mx.operator.register('rpn_fpn_ohem3') -class RPNFPNOHEM3Prop(mx.operator.CustomOpProp): - def __init__(self, stride=0, network='', dataset='', prefix=''): - super(RPNFPNOHEM3Prop, self).__init__(need_top_grad=False) - self.stride = stride - self.network=network - self.dataset=dataset - self.prefix = prefix - - def list_arguments(self): - return ['cls_score', 'labels'] - - def list_outputs(self): - return ['labels_ohem', 'anchor_weight', 'valid_count'] - - def infer_shape(self, in_shape): - labels_shape = in_shape[1] - #print('in_rpn_ohem', in_shape[0], in_shape[1], in_shape[2], file=sys.stderr) - anchor_weight_shape = [labels_shape[0], labels_shape[1], 1] - #print('in_rpn_ohem', labels_shape, anchor_weight_shape) - - return in_shape, \ - [labels_shape, anchor_weight_shape, [labels_shape[0], 1]] - - def create_operator(self, ctx, shapes, dtypes): - return RPNFPNOHEM3Operator(self.stride, self.network, self.dataset, self.prefix) - - def declare_backward_dependency(self, out_grad, in_data, out_data): - return [] - - diff --git a/RetinaFace/rcnn/core/callback.py b/RetinaFace/rcnn/core/callback.py deleted file mode 100644 index 317d5cd..0000000 --- a/RetinaFace/rcnn/core/callback.py +++ /dev/null @@ -1,13 +0,0 @@ -import mxnet as mx - - -def do_checkpoint(prefix, means, stds): - def _callback(iter_no, sym, arg, aux): - if 'bbox_pred_weight' in arg: - arg['bbox_pred_weight_test'] = (arg['bbox_pred_weight'].T * mx.nd.array(stds)).T - arg['bbox_pred_bias_test'] = arg['bbox_pred_bias'] * mx.nd.array(stds) + mx.nd.array(means) - mx.model.save_checkpoint(prefix, iter_no + 1, sym, arg, aux) - if 'bbox_pred_weight' in arg: - arg.pop('bbox_pred_weight_test') - arg.pop('bbox_pred_bias_test') - return _callback diff --git a/RetinaFace/rcnn/dataset/retinaface.py b/RetinaFace/rcnn/dataset/retinaface.py deleted file mode 100644 index 53fe1ab..0000000 --- a/RetinaFace/rcnn/dataset/retinaface.py +++ /dev/null @@ -1,186 +0,0 @@ -from __future__ import print_function -try: - import cPickle as pickle -except ImportError: - import pickle -import cv2 -import os -import numpy as np -import json -#from PIL import Image - -from ..logger import logger -from .imdb import IMDB -from .ds_utils import unique_boxes, filter_small_boxes -from ..config import config - -class retinaface(IMDB): - def __init__(self, image_set, root_path, data_path): - super(retinaface, self).__init__('retinaface', image_set, root_path, data_path) - #assert image_set=='train' - - split = image_set - self._split = image_set - self._image_set = image_set - - - self.root_path = root_path - self.data_path = data_path - - - self._dataset_path = self.data_path - self._imgs_path = os.path.join(self._dataset_path, image_set, 'images') - self._fp_bbox_map = {} - label_file = os.path.join(self._dataset_path, image_set, 'label.txt') - name = None - for line in open(label_file, 'r'): - line = line.strip() - if line.startswith('#'): - name = line[1:].strip() - self._fp_bbox_map[name] = [] - continue - assert name is not None - assert name in self._fp_bbox_map - self._fp_bbox_map[name].append(line) - print('origin image size', len(self._fp_bbox_map)) - - #self.num_images = len(self._image_paths) - #self._image_index = range(len(self._image_paths)) - self.classes = ['bg', 'face'] - self.num_classes = len(self.classes) - - - def gt_roidb(self): - cache_file = os.path.join(self.cache_path, '{}_{}_gt_roidb.pkl'.format(self.name, self._split)) - if os.path.exists(cache_file): - with open(cache_file, 'rb') as fid: - roidb = pickle.load(fid) - print('{} gt roidb loaded from {}'.format(self.name, cache_file)) - self.num_images = len(roidb) - return roidb - - roidb = [] - max_num_boxes = 0 - nonattr_box_num = 0 - landmark_num = 0 - - pp = 0 - for fp in self._fp_bbox_map: - pp += 1 - if pp%1000==0: - print('loading', pp) - if self._split=='test': - image_path = os.path.join(self._imgs_path, fp) - roi = {'image': image_path} - roidb.append(roi) - continue - boxes = np.zeros([len(self._fp_bbox_map[fp]), 4], np.float) - landmarks = np.zeros([len(self._fp_bbox_map[fp]), 5, 3], np.float) - blur = np.zeros((len(self._fp_bbox_map[fp]),), np.float) - boxes_mask = [] - - gt_classes = np.ones([len(self._fp_bbox_map[fp])], np.int32) - overlaps = np.zeros([len(self._fp_bbox_map[fp]), 2], np.float) - - imsize = cv2.imread(os.path.join(self._imgs_path, fp)).shape[0:2][::-1] - ix = 0 - - for aline in self._fp_bbox_map[fp]: - #imsize = Image.open(os.path.join(self._imgs_path, fp)).size - values = [float(x) for x in aline.strip().split()] - bbox = [values[0], values[1], values[0]+values[2], values[1]+values[3]] - - x1 = bbox[0] - y1 = bbox[1] - x2 = min(imsize[0], bbox[2]) - y2 = min(imsize[1], bbox[3]) - if x1>=x2 or y1>=y2: - continue - - if config.BBOX_MASK_THRESH>0: - if (x2 - x1) < config.BBOX_MASK_THRESH or y2 - y1 < config.BBOX_MASK_THRESH: - boxes_mask.append(np.array([x1, y1, x2, y2], np.float)) - continue - if (x2 - x1) < config.TRAIN.MIN_BOX_SIZE or y2 - y1 < config.TRAIN.MIN_BOX_SIZE: - continue - - boxes[ix, :] = np.array([x1, y1, x2, y2], np.float) - if self._split=='train': - landmark = np.array( values[4:19], dtype=np.float32 ).reshape((5,3)) - for li in range(5): - #print(landmark) - if landmark[li][0]==-1. and landmark[li][1]==-1.: #missing landmark - assert landmark[li][2]==-1 - else: - assert landmark[li][2]>=0 - if li==0: - landmark_num+=1 - if landmark[li][2]==0.0:#visible - landmark[li][2] = 1.0 - else: - landmark[li][2] = 0.0 - - landmarks[ix] = landmark - - blur[ix] = values[19] - #print(aline, blur[ix]) - if blur[ix]<0.0: - blur[ix] = 0.3 - nonattr_box_num+=1 - - cls = int(1) - gt_classes[ix] = cls - overlaps[ix, cls] = 1.0 - ix += 1 - max_num_boxes = max(max_num_boxes, ix) - #overlaps = scipy.sparse.csr_matrix(overlaps) - if self._split=='train' and ix==0: - continue - boxes = boxes[:ix,:] - landmarks = landmarks[:ix,:,:] - blur = blur[:ix] - gt_classes = gt_classes[:ix] - overlaps = overlaps[:ix,:] - image_path = os.path.join(self._imgs_path, fp) - with open(image_path, 'rb') as fin: - stream = fin.read() - stream = np.fromstring(stream, dtype=np.uint8) - - roi = { - 'image': image_path, - 'stream': stream, - 'height': imsize[1], - 'width': imsize[0], - 'boxes': boxes, - 'landmarks': landmarks, - 'blur': blur, - 'gt_classes': gt_classes, - 'gt_overlaps': overlaps, - 'max_classes': overlaps.argmax(axis=1), - 'max_overlaps': overlaps.max(axis=1), - 'flipped': False, - } - if len(boxes_mask)>0: - boxes_mask = np.array(boxes_mask) - roi['boxes_mask'] = boxes_mask - roidb.append(roi) - for roi in roidb: - roi['max_num_boxes'] = max_num_boxes - self.num_images = len(roidb) - print('roidb size', len(roidb)) - print('non attr box num', nonattr_box_num) - print('landmark num', landmark_num) - with open(cache_file, 'wb') as fid: - pickle.dump(roidb, fid, pickle.HIGHEST_PROTOCOL) - print('wrote gt roidb to {}'.format(cache_file)) - - return roidb - - def write_detections(self, all_boxes, output_dir='./output/'): - pass - - - def evaluate_detections(self, all_boxes, output_dir='./output/',method_name='insightdetection'): - pass - - diff --git a/RetinaFace/rcnn/io/image.py b/RetinaFace/rcnn/io/image.py deleted file mode 100644 index 32158f9..0000000 --- a/RetinaFace/rcnn/io/image.py +++ /dev/null @@ -1,808 +0,0 @@ -from __future__ import print_function -import numpy as np -import cv2 -import os -import math -import sys -import random -from ..config import config - -def brightness_aug(src, x): - alpha = 1.0 + random.uniform(-x, x) - src *= alpha - return src - -def contrast_aug(src, x): - alpha = 1.0 + random.uniform(-x, x) - coef = np.array([[[0.299, 0.587, 0.114]]]) - gray = src * coef - gray = (3.0 * (1.0 - alpha) / gray.size) * np.sum(gray) - src *= alpha - src += gray - return src - -def saturation_aug(src, x): - alpha = 1.0 + random.uniform(-x, x) - coef = np.array([[[0.299, 0.587, 0.114]]]) - gray = src * coef - gray = np.sum(gray, axis=2, keepdims=True) - gray *= (1.0 - alpha) - src *= alpha - src += gray - return src - -def color_aug(img, x): - if config.COLOR_MODE>1: - augs = [brightness_aug, contrast_aug, saturation_aug] - random.shuffle(augs) - else: - augs = [brightness_aug] - for aug in augs: - #print(img.shape) - img = aug(img, x) - #print(img.shape) - return img - -def get_image(roidb, scale=False): - """ - preprocess image and return processed roidb - :param roidb: a list of roidb - :return: list of img as in mxnet format - roidb add new item['im_info'] - 0 --- x (width, second dim of im) - | - y (height, first dim of im) - """ - num_images = len(roidb) - processed_ims = [] - processed_roidb = [] - for i in range(num_images): - roi_rec = roidb[i] - if 'stream' in roi_rec: - im = cv2.imdecode(roi_rec['stream'], cv2.IMREAD_COLOR) - else: - assert os.path.exists(roi_rec['image']), '{} does not exist'.format(roi_rec['image']) - im = cv2.imread(roi_rec['image']) - if roidb[i]['flipped']: - im = im[:, ::-1, :] - new_rec = roi_rec.copy() - if scale: - scale_range = config.TRAIN.SCALE_RANGE - im_scale = np.random.uniform(scale_range[0], scale_range[1]) - im = cv2.resize(im, None, None, fx=im_scale, fy=im_scale, interpolation=cv2.INTER_LINEAR) - elif not config.ORIGIN_SCALE: - scale_ind = random.randrange(len(config.SCALES)) - target_size = config.SCALES[scale_ind][0] - max_size = config.SCALES[scale_ind][1] - im, im_scale = resize(im, target_size, max_size, stride=config.IMAGE_STRIDE) - else: - im_scale = 1.0 - im_tensor = transform(im, config.PIXEL_MEANS, config.PIXEL_STDS) - if 'boxes_mask' in roi_rec: - im = im.astype(np.float32) - boxes_mask = roi_rec['boxes_mask'].copy() * im_scale - boxes_mask = boxes_mask.astype(np.int) - for j in range(boxes_mask.shape[0]): - m = boxes_mask[j] - im_tensor[:,:,m[1]:m[3],m[0]:m[2]] = 0.0 - #print('find mask', m, file=sys.stderr) - processed_ims.append(im_tensor) - new_rec['boxes'] = roi_rec['boxes'].copy() * im_scale - if config.TRAIN.IMAGE_ALIGN>0: - if im_tensor.shape[2]%config.TRAIN.IMAGE_ALIGN!=0 or im_tensor.shape[3]%config.TRAIN.IMAGE_ALIGN!=0: - new_height = math.ceil(float(im_tensor.shape[2])/config.TRAIN.IMAGE_ALIGN)*config.TRAIN.IMAGE_ALIGN - new_width = math.ceil(float(im_tensor.shape[3])/config.TRAIN.IMAGE_ALIGN)*config.TRAIN.IMAGE_ALIGN - new_im_tensor = np.zeros((1, 3, int(new_height), int(new_width))) - new_im_tensor[:,:,0:im_tensor.shape[2],0:im_tensor.shape[3]] = im_tensor - print(im_tensor.shape, new_im_tensor.shape, file=sys.stderr) - im_tensor = new_im_tensor - #print('boxes', new_rec['boxes'], file=sys.stderr) - im_info = [im_tensor.shape[2], im_tensor.shape[3], im_scale] - new_rec['im_info'] = im_info - processed_roidb.append(new_rec) - return processed_ims, processed_roidb - -TMP_ID = -1 -#bakup method -def __get_crop_image(roidb): - """ - preprocess image and return processed roidb - :param roidb: a list of roidb - :return: list of img as in mxnet format - roidb add new item['im_info'] - 0 --- x (width, second dim of im) - | - y (height, first dim of im) - """ - #roidb and each roi_rec can not be changed as it will be reused in next epoch - num_images = len(roidb) - processed_ims = [] - processed_roidb = [] - for i in range(num_images): - roi_rec = roidb[i] - if 'stream' in roi_rec: - im = cv2.imdecode(roi_rec['stream'], cv2.IMREAD_COLOR) - else: - assert os.path.exists(roi_rec['image']), '{} does not exist'.format(roi_rec['image']) - im = cv2.imread(roi_rec['image']) - if roidb[i]['flipped']: - im = im[:, ::-1, :] - if 'boxes_mask' in roi_rec: - #im = im.astype(np.float32) - boxes_mask = roi_rec['boxes_mask'].copy() - boxes_mask = boxes_mask.astype(np.int) - for j in range(boxes_mask.shape[0]): - m = boxes_mask[j] - im[m[1]:m[3],m[0]:m[2],:] = 0 - #print('find mask', m, file=sys.stderr) - new_rec = roi_rec.copy() - - - #choose one gt randomly - SIZE = config.SCALES[0][0] - TARGET_BOX_SCALES = np.array([16,32,64,128,256,512]) - assert roi_rec['boxes'].shape[0]>0 - candidates = [] - for i in range(roi_rec['boxes'].shape[0]): - box = roi_rec['boxes'][i] - box_size = max(box[2]-box[0], box[3]-box[1]) - if box_sizeim.shape[1] or box[3]>im.shape[0]: - # continue; - candidates.append(i) - assert len(candidates)>0 - box_ind = random.choice(candidates) - box = roi_rec['boxes'][box_ind] - box_size = max(box[2]-box[0], box[3]-box[1]) - dist = np.abs(TARGET_BOX_SCALES - box_size) - nearest = np.argmin(dist) - target_ind = random.randrange(min(len(TARGET_BOX_SCALES), nearest+2)) - target_box_size = TARGET_BOX_SCALES[target_ind] - im_scale = float(target_box_size) / box_size - #min_scale = float(SIZE)/np.min(im.shape[0:2]) - #if im_scale=im.shape[1] or center[1]>=im.shape[0]: - continue - if box_size0 - DEBUG = True - if DEBUG: - global TMP_ID - if TMP_ID<10: - tim = im.copy() - for i in range(new_rec['boxes'].shape[0]): - box = new_rec['boxes'][i].copy().astype(np.int) - cv2.rectangle(tim, (box[0], box[1]), (box[2], box[3]), (255, 0, 0), 1) - filename = './trainimages/train%d.png' % TMP_ID - TMP_ID+=1 - cv2.imwrite(filename, tim) - - im_tensor = transform(im, config.PIXEL_MEANS, config.PIXEL_STDS, config.PIXEL_SCALE) - - processed_ims.append(im_tensor) - #print('boxes', new_rec['boxes'], file=sys.stderr) - im_info = [im_tensor.shape[2], im_tensor.shape[3], im_scale] - new_rec['im_info'] = im_info - processed_roidb.append(new_rec) - return processed_ims, processed_roidb - -def expand_bboxes(bboxes, - image_width, - image_height, - expand_left=2., - expand_up=2., - expand_right=2., - expand_down=2.): - """ - Expand bboxes, expand 2 times by defalut. - """ - expand_boxes = [] - for bbox in bboxes: - xmin = bbox[0] - ymin = bbox[1] - xmax = bbox[2] - ymax = bbox[3] - w = xmax - xmin - h = ymax - ymin - ex_xmin = max(xmin - w / expand_left, 0.) - ex_ymin = max(ymin - h / expand_up, 0.) - ex_xmax = min(xmax + w / expand_right, image_width) - ex_ymax = min(ymax + h / expand_down, image_height) - expand_boxes.append([ex_xmin, ex_ymin, ex_xmax, ex_ymax]) - return expand_boxes - -def get_crop_image1(roidb): - """ - preprocess image and return processed roidb - :param roidb: a list of roidb - :return: list of img as in mxnet format - roidb add new item['im_info'] - 0 --- x (width, second dim of im) - | - y (height, first dim of im) - """ - #roidb and each roi_rec can not be changed as it will be reused in next epoch - num_images = len(roidb) - processed_ims = [] - processed_roidb = [] - for i in range(num_images): - roi_rec = roidb[i] - if 'stream' in roi_rec: - im = cv2.imdecode(roi_rec['stream'], cv2.IMREAD_COLOR) - else: - assert os.path.exists(roi_rec['image']), '{} does not exist'.format(roi_rec['image']) - im = cv2.imread(roi_rec['image']) - if roidb[i]['flipped']: - im = im[:, ::-1, :] - if 'boxes_mask' in roi_rec: - #im = im.astype(np.float32) - boxes_mask = roi_rec['boxes_mask'].copy() - boxes_mask = boxes_mask.astype(np.int) - for j in range(boxes_mask.shape[0]): - m = boxes_mask[j] - im[m[1]:m[3],m[0]:m[2],:] = 127 - #print('find mask', m, file=sys.stderr) - SIZE = config.SCALES[0][0] - PRE_SCALES = [0.3, 0.45, 0.6, 0.8, 1.0] - #PRE_SCALES = [0.3, 0.45, 0.6, 0.8, 1.0, 0.8, 1.0, 0.8, 1.0] - _scale = random.choice(PRE_SCALES) - #_scale = np.random.uniform(PRE_SCALES[0], PRE_SCALES[-1]) - size = int(np.min(im.shape[0:2])*_scale) - #size = int(np.round(_scale*np.min(im.shape[0:2]))) - im_scale = float(SIZE)/size - #origin_im_scale = im_scale - #size = np.round(np.min(im.shape[0:2])*im_scale) - #im_scale *= (float(SIZE)/size) - origin_shape = im.shape - if _scale>10.0: #avoid im.size=SIZE and im.shape[1]>=SIZE - #print('image size', origin_shape, _scale, SIZE, size, im_scale) - - new_rec = roi_rec.copy() - new_rec['boxes'] = roi_rec['boxes'].copy() * im_scale - if config.FACE_LANDMARK: - new_rec['landmarks'] = roi_rec['landmarks'].copy() - new_rec['landmarks'][:,:,0:2] *= im_scale - retry = 0 - LIMIT = 25 - size = SIZE - while retry=im_new.shape[1] or centery>=im_new.shape[0]: - continue - if box_size0 or retry==LIMIT-1: - im = im_new - new_rec['boxes'] = np.array(valid_boxes) - new_rec['gt_classes'] = new_rec['gt_classes'][valid] - if config.FACE_LANDMARK: - new_rec['landmarks'] = np.array(valid_landmarks) - if config.HEAD_BOX: - face_box = new_rec['boxes'] - head_box = expand_bboxes(face_box, image_width=im.shape[1], image_height=im.shape[0]) - new_rec['boxes_head'] = np.array(head_box) - break - - retry+=1 - - if config.COLOR_MODE>0 and config.COLOR_JITTERING>0.0: - im = im.astype(np.float32) - im = color_aug(im, config.COLOR_JITTERING) - - #assert np.all(new_rec['landmarks'][:,10]>0.0) - global TMP_ID - if TMP_ID>=0 and TMP_ID<10: - tim = im.copy().astype(np.uint8) - for i in range(new_rec['boxes'].shape[0]): - box = new_rec['boxes'][i].copy().astype(np.int) - cv2.rectangle(tim, (box[0], box[1]), (box[2], box[3]), (255, 0, 0), 1) - print('draw box:', box) - if config.FACE_LANDMARK: - for i in range(new_rec['landmarks'].shape[0]): - landmark = new_rec['landmarks'][i].copy() - if landmark[0][2]<0: - print('zero', landmark) - continue - landmark = landmark.astype(np.int) - print('draw landmark', landmark) - for k in range(5): - color = (0, 0, 255) - if k==0 or k==3: - color = (0, 255, 0) - pp = (landmark[k][0], landmark[k][1]) - cv2.circle(tim, (pp[0], pp[1]), 1, color, 2) - filename = './trainimages/train%d.png' % TMP_ID - print('write', filename) - cv2.imwrite(filename, tim) - TMP_ID+=1 - - im_tensor = transform(im, config.PIXEL_MEANS, config.PIXEL_STDS, config.PIXEL_SCALE) - - processed_ims.append(im_tensor) - #print('boxes', new_rec['boxes'], file=sys.stderr) - im_info = [im_tensor.shape[2], im_tensor.shape[3], im_scale] - new_rec['im_info'] = np.array(im_info, dtype=np.float32) - processed_roidb.append(new_rec) - return processed_ims, processed_roidb - -def get_crop_image2(roidb): - """ - preprocess image and return processed roidb - :param roidb: a list of roidb - :return: list of img as in mxnet format - roidb add new item['im_info'] - 0 --- x (width, second dim of im) - | - y (height, first dim of im) - """ - #roidb and each roi_rec can not be changed as it will be reused in next epoch - num_images = len(roidb) - processed_ims = [] - processed_roidb = [] - for i in range(num_images): - roi_rec = roidb[i] - if 'stream' in roi_rec: - im = cv2.imdecode(roi_rec['stream'], cv2.IMREAD_COLOR) - else: - assert os.path.exists(roi_rec['image']), '{} does not exist'.format(roi_rec['image']) - im = cv2.imread(roi_rec['image']) - if roidb[i]['flipped']: - im = im[:, ::-1, :] - if 'boxes_mask' in roi_rec: - #im = im.astype(np.float32) - boxes_mask = roi_rec['boxes_mask'].copy() - boxes_mask = boxes_mask.astype(np.int) - for j in range(boxes_mask.shape[0]): - m = boxes_mask[j] - im[m[1]:m[3],m[0]:m[2],:] = 0 - #print('find mask', m, file=sys.stderr) - SIZE = config.SCALES[0][0] - scale_array = np.array([16,32,64,128,256,512], dtype=np.float32) - candidates = [] - for i in range(roi_rec['boxes'].shape[0]): - box = roi_rec['boxes'][i] - box_size = max(box[2]-box[0], box[3]-box[1]) - if box_sizeim.shape[1] or box[3]>im.shape[0]: - # continue; - candidates.append(i) - assert len(candidates)>0 - box_ind = random.choice(candidates) - box = roi_rec['boxes'][box_ind] - width = box[2]-box[0] - height = box[3]-box[1] - wid = width - hei = height - resize_width, resize_height = config.SCALES[0] - image_width = im.shape[0] - image_height = im.shape[1] - area = width*height - range_size = 0 - for scale_ind in range(0, len(scale_array) - 1): - if area > scale_array[scale_ind] ** 2 and area < \ - scale_array[scale_ind + 1] ** 2: - range_size = scale_ind + 1 - break - - if area > scale_array[len(scale_array) - 2]**2: - range_size = len(scale_array) - 2 - scale_choose = 0.0 - if range_size == 0: - rand_idx_size = 0 - else: - # np.random.randint range: [low, high) - rng_rand_size = np.random.randint(0, range_size + 1) - rand_idx_size = rng_rand_size % (range_size + 1) - - if rand_idx_size == range_size: - min_resize_val = scale_array[rand_idx_size] / 2.0 - max_resize_val = min(2.0 * scale_array[rand_idx_size], - 2 * math.sqrt(wid * hei)) - scale_choose = random.uniform(min_resize_val, max_resize_val) - else: - min_resize_val = scale_array[rand_idx_size] / 2.0 - max_resize_val = 2.0 * scale_array[rand_idx_size] - scale_choose = random.uniform(min_resize_val, max_resize_val) - - sample_bbox_size = wid * resize_width / scale_choose - - w_off_orig = 0.0 - h_off_orig = 0.0 - if sample_bbox_size < max(image_height, image_width): - if wid <= sample_bbox_size: - w_off_orig = np.random.uniform(xmin + wid - sample_bbox_size, - xmin) - else: - w_off_orig = np.random.uniform(xmin, - xmin + wid - sample_bbox_size) - - if hei <= sample_bbox_size: - h_off_orig = np.random.uniform(ymin + hei - sample_bbox_size, - ymin) - else: - h_off_orig = np.random.uniform(ymin, - ymin + hei - sample_bbox_size) - - else: - w_off_orig = np.random.uniform(image_width - sample_bbox_size, 0.0) - h_off_orig = np.random.uniform(image_height - sample_bbox_size, 0.0) - - w_off_orig = math.floor(w_off_orig) - h_off_orig = math.floor(h_off_orig) - - # Figure out top left coordinates. - w_off = 0.0 - h_off = 0.0 - w_off = float(w_off_orig / image_width) - h_off = float(h_off_orig / image_height) - im_new = im[up:(up+size), left:(left+size), :] - - sampled_bbox = bbox(w_off, h_off, - w_off + float(sample_bbox_size / image_width), - h_off + float(sample_bbox_size / image_height)) - return sampled_bbox - - box_size = max(box[2]-box[0], box[3]-box[1]) - dist = np.abs(TARGET_BOX_SCALES - box_size) - nearest = np.argmin(dist) - target_ind = random.randrange(min(len(TARGET_BOX_SCALES), nearest+2)) - target_box_size = TARGET_BOX_SCALES[target_ind] - im_scale = float(target_box_size) / box_size - PRE_SCALES = [0.3, 0.45, 0.6, 0.8, 1.0] - _scale = random.choice(PRE_SCALES) - #_scale = np.random.uniform(PRE_SCALES[0], PRE_SCALES[-1]) - size = int(np.round(_scale*np.min(im.shape[0:2]))) - im_scale = float(SIZE)/size - #origin_im_scale = im_scale - #size = np.round(np.min(im.shape[0:2])*im_scale) - #im_scale *= (float(SIZE)/size) - origin_shape = im.shape - if _scale>10.0: #avoid im.size=SIZE and im.shape[1]>=SIZE - - new_rec = roi_rec.copy() - new_rec['boxes'] = roi_rec['boxes'].copy() * im_scale - if config.FACE_LANDMARK: - new_rec['landmarks'] = roi_rec['landmarks'].copy() * im_scale - retry = 0 - LIMIT = 25 - size = SIZE - while retry=im_new.shape[1] or centery>=im_new.shape[0]: - continue - if box_size0 or retry==LIMIT-1: - im = im_new - new_rec['boxes'] = np.array(valid_boxes) - new_rec['gt_classes'] = new_rec['gt_classes'][valid] - if config.FACE_LANDMARK: - new_rec['landmarks'] = np.array(valid_landmarks) - break - - retry+=1 - - if config.COLOR_JITTERING>0.0: - im = im.astype(np.float32) - im = color_aug(im, config.COLOR_JITTERING) - - #assert np.all(new_rec['landmarks'][:,10]>0.0) - global TMP_ID - if TMP_ID>=0 and TMP_ID<10: - tim = im.copy().astype(np.uint8) - for i in range(new_rec['boxes'].shape[0]): - box = new_rec['boxes'][i].copy().astype(np.int) - cv2.rectangle(tim, (box[0], box[1]), (box[2], box[3]), (255, 0, 0), 1) - print('draw box:', box) - if config.FACE_LANDMARK: - for i in range(new_rec['landmarks'].shape[0]): - landmark = new_rec['landmarks'][i].copy() - if landmark[10]==0.0: - print('zero', landmark) - continue - landmark = landmark.astype(np.int) - print('draw landmark', landmark) - for k in range(5): - color = (0, 0, 255) - if k==0 or k==3: - color = (0, 255, 0) - pp = (landmark[k*2], landmark[1+k*2]) - cv2.circle(tim, (pp[0], pp[1]), 1, color, 2) - filename = './trainimages/train%d.png' % TMP_ID - print('write', filename) - cv2.imwrite(filename, tim) - TMP_ID+=1 - - im_tensor = transform(im, config.PIXEL_MEANS, config.PIXEL_STDS, config.PIXEL_SCALE) - - processed_ims.append(im_tensor) - #print('boxes', new_rec['boxes'], file=sys.stderr) - im_info = [im_tensor.shape[2], im_tensor.shape[3], im_scale] - new_rec['im_info'] = np.array(im_info, dtype=np.float32) - processed_roidb.append(new_rec) - return processed_ims, processed_roidb - -def do_mixup(im1, roidb1, im2, roidb2): - im = (im1+im2)/2.0 - roidb = {} - #print(roidb1.keys()) - #for k in roidb1: - for k in ['boxes', 'landmarks', 'gt_classes', 'im_info']: - v1 = roidb1[k] - v2 = roidb2[k] - if k!='im_info': - #print('try', k, v1.shape, v2.shape) - if v1.shape[0]>0 and v2.shape[0]>0: - v = np.concatenate( (v1, v2), axis=0 ) - else: - v = v1 - else: - v = v1 - #print(k, v1.shape, v2.shape, v.shape) - roidb[k] = v - return im, roidb - -def get_crop_image(roidb): - ims, roidbs = get_crop_image1(roidb) - if config.MIXUP>0.0 and np.random.random()=i: - j+=1 - im, roidb = do_mixup(im, roidb, ims[j], roidbs[j]) - ims[i] = im - roidbs[i] = roidb - return ims, roidbs - -def resize(im, target_size, max_size, stride=0, min_size=0): - """ - only resize input image to target size and return scale - :param im: BGR image input by opencv - :param target_size: one dimensional size (the short side) - :param max_size: one dimensional max size (the long side) - :param stride: if given, pad the image to designated stride - :return: - """ - im_shape = im.shape - im_size_min = np.min(im_shape[0:2]) - im_size_max = np.max(im_shape[0:2]) - im_scale = float(target_size) / float(im_size_min) - # prevent bigger axis from being more than max_size: - if np.round(im_scale * im_size_max) > max_size: - im_scale = float(max_size) / float(im_size_max) - if min_size>0 and np.round(im_scale*im_size_min) 0: - gt_inds = np.where(roidb[0]['gt_classes'] != 0)[0] - gt_boxes = np.empty((roidb[0]['boxes'].shape[0], 5), dtype=np.float32) - gt_boxes[:, 0:4] = roidb[0]['boxes'][gt_inds, :] - gt_boxes[:, 4] = roidb[0]['gt_classes'][gt_inds] - else: - gt_boxes = np.empty((0, 5), dtype=np.float32) - - data = {'data': im_array, - 'im_info': im_info} - label = {'gt_boxes': gt_boxes} - - return data, label - -def get_crop_batch(roidb): - """ - prototype for rpn batch: data, im_info, gt_boxes - :param roidb: ['image', 'flipped'] + ['gt_boxes', 'boxes', 'gt_classes'] - :return: data, label - """ - #assert len(roidb) == 1, 'Single batch only' - data_list = [] - label_list = [] - imgs, roidb = get_crop_image(roidb) - assert len(imgs)==len(roidb) - for i in range(len(imgs)): - im_array = imgs[i] - im_info = np.array([roidb[i]['im_info']], dtype=np.float32) - - # gt boxes: (x1, y1, x2, y2, cls) - if roidb[i]['gt_classes'].size > 0: - gt_inds = np.where(roidb[i]['gt_classes'] != 0)[0] - gt_boxes = np.empty((roidb[i]['boxes'].shape[0], 5), dtype=np.float32) - gt_boxes[:, 0:4] = roidb[i]['boxes'][gt_inds, :] - gt_boxes[:, 4] = roidb[i]['gt_classes'][gt_inds] - if config.USE_BLUR: - gt_blur = roidb[i]['blur'] - if config.FACE_LANDMARK: - #gt_landmarks = np.empty((roidb[i]['landmarks'].shape[0], 11), dtype=np.float32) - gt_landmarks = roidb[i]['landmarks'][gt_inds,:,:] - if config.HEAD_BOX: - gt_boxes_head = np.empty((roidb[i]['boxes_head'].shape[0], 5), dtype=np.float32) - gt_boxes_head[:, 0:4] = roidb[i]['boxes_head'][gt_inds, :] - gt_boxes_head[:, 4] = roidb[i]['gt_classes'][gt_inds] - else: - gt_boxes = np.empty((0, 5), dtype=np.float32) - if config.USE_BLUR: - gt_blur = np.empty((0,), dtype=np.float32) - if config.FACE_LANDMARK: - gt_landmarks = np.empty((0, 5, 3), dtype=np.float32) - if config.HEAD_BOX: - gt_boxes_head = np.empty((0, 5), dtype=np.float32) - - data = {'data': im_array, - 'im_info': im_info} - label = {'gt_boxes': gt_boxes} - if config.USE_BLUR: - label['gt_blur'] = gt_blur - if config.FACE_LANDMARK: - label['gt_landmarks'] = gt_landmarks - if config.HEAD_BOX: - label['gt_boxes_head'] = gt_boxes_head - data_list.append(data) - label_list.append(label) - - return data_list, label_list - -def assign_anchor_fpn(feat_shape, gt_label, im_info, landmark=False, prefix='face', select_stride=0): - """ - assign ground truth boxes to anchor positions - :param feat_shape: infer output shape - :param gt_boxes: assign ground truth - :param im_info: filter out anchors overlapped with edges - :return: tuple - labels: of shape (batch_size, 1) <- (batch_size, num_anchors, feat_height, feat_width) - bbox_targets: of shape (batch_size, num_anchors * 4, feat_height, feat_width) - bbox_weights: mark the assigned anchors - """ - def _unmap(data, count, inds, fill=0): - """" unmap a subset inds of data into original data of size count """ - if len(data.shape) == 1: - ret = np.empty((count,), dtype=np.float32) - ret.fill(fill) - ret[inds] = data - else: - ret = np.empty((count,) + data.shape[1:], dtype=np.float32) - ret.fill(fill) - ret[inds, :] = data - return ret - - global STAT - DEBUG = False - - im_info = im_info[0] - gt_boxes = gt_label['gt_boxes'] - # clean up boxes - nonneg = np.where(gt_boxes[:, 4] != -1)[0] - gt_boxes = gt_boxes[nonneg] - if config.USE_BLUR: - gt_blur = gt_label['gt_blur'] - gt_blur = gt_blur[nonneg] - if landmark: - gt_landmarks = gt_label['gt_landmarks'] - gt_landmarks = gt_landmarks[nonneg] - assert gt_boxes.shape[0]==gt_landmarks.shape[0] - #scales = np.array(scales, dtype=np.float32) - feat_strides = config.RPN_FEAT_STRIDE - bbox_pred_len = 4 - landmark_pred_len = 10 - if config.USE_BLUR: - gt_boxes[:,4] = gt_blur - bbox_pred_len = 5 - if config.USE_OCCLUSION: - landmark_pred_len = 15 - - anchors_list = [] - anchors_num_list = [] - inds_inside_list = [] - feat_infos = [] - A_list = [] - for i in range(len(feat_strides)): - stride = feat_strides[i] - sstride = str(stride) - base_size = config.RPN_ANCHOR_CFG[sstride]['BASE_SIZE'] - allowed_border = config.RPN_ANCHOR_CFG[sstride]['ALLOWED_BORDER'] - ratios = config.RPN_ANCHOR_CFG[sstride]['RATIOS'] - scales = config.RPN_ANCHOR_CFG[sstride]['SCALES'] - base_anchors = generate_anchors(base_size=base_size, ratios=list(ratios), scales=np.array(scales, dtype=np.float32), stride = stride, dense_anchor = config.DENSE_ANCHOR) - num_anchors = base_anchors.shape[0] - feat_height, feat_width = feat_shape[i][-2:] - feat_stride = feat_strides[i] - feat_infos.append([feat_height, feat_width]) - - A = num_anchors - A_list.append(A) - K = feat_height * feat_width - - all_anchors = anchors_plane(feat_height, feat_width, feat_stride, base_anchors) - all_anchors = all_anchors.reshape((K * A, 4)) - #print('anchor0', stride, all_anchors[0]) - - total_anchors = int(K * A) - anchors_num_list.append(total_anchors) - # only keep anchors inside the image - inds_inside = np.where((all_anchors[:, 0] >= -allowed_border) & - (all_anchors[:, 1] >= -allowed_border) & - (all_anchors[:, 2] < im_info[1] + allowed_border) & - (all_anchors[:, 3] < im_info[0] + allowed_border))[0] - if DEBUG: - print('total_anchors', total_anchors) - print('inds_inside', len(inds_inside)) - - # keep only inside anchors - anchors = all_anchors[inds_inside, :] - #print('AA', anchors.shape, len(inds_inside)) - - anchors_list.append(anchors) - inds_inside_list.append(inds_inside) - - # Concat anchors from each level - anchors = np.concatenate(anchors_list) - for i in range(1, len(inds_inside_list)): - inds_inside_list[i] = inds_inside_list[i] + sum(anchors_num_list[:i]) - inds_inside = np.concatenate(inds_inside_list) - total_anchors = sum(anchors_num_list) - #print('total_anchors', anchors.shape[0], len(inds_inside), file=sys.stderr) - - # label: 1 is positive, 0 is negative, -1 is dont care - labels = np.empty((len(inds_inside),), dtype=np.float32) - labels.fill(-1) - #print('BB', anchors.shape, len(inds_inside)) - #print('gt_boxes', gt_boxes.shape, file=sys.stderr) - - if gt_boxes.size > 0: - # overlap between the anchors and the gt boxes - # overlaps (ex, gt) - overlaps = bbox_overlaps(anchors.astype(np.float), gt_boxes.astype(np.float)) - argmax_overlaps = overlaps.argmax(axis=1) - #print('AAA', argmax_overlaps.shape) - max_overlaps = overlaps[np.arange(len(inds_inside)), argmax_overlaps] - gt_argmax_overlaps = overlaps.argmax(axis=0) - gt_max_overlaps = overlaps[gt_argmax_overlaps, np.arange(overlaps.shape[1])] - gt_argmax_overlaps = np.where(overlaps == gt_max_overlaps)[0] - - if not config.TRAIN.RPN_CLOBBER_POSITIVES: - # assign bg labels first so that positive labels can clobber them - labels[max_overlaps < config.TRAIN.RPN_NEGATIVE_OVERLAP] = 0 - - # fg label: for each gt, anchor with highest overlap - if config.TRAIN.RPN_FORCE_POSITIVE: - labels[gt_argmax_overlaps] = 1 - - # fg label: above threshold IoU - labels[max_overlaps >= config.TRAIN.RPN_POSITIVE_OVERLAP] = 1 - - if config.TRAIN.RPN_CLOBBER_POSITIVES: - # assign bg labels last so that negative labels can clobber positives - labels[max_overlaps < config.TRAIN.RPN_NEGATIVE_OVERLAP] = 0 - else: - labels[:] = 0 - fg_inds = np.where(labels == 1)[0] - #print('fg count', len(fg_inds)) - - # subsample positive labels if we have too many - if config.TRAIN.RPN_ENABLE_OHEM==0: - fg_inds = np.where(labels == 1)[0] - num_fg = int(config.TRAIN.RPN_FG_FRACTION * config.TRAIN.RPN_BATCH_SIZE) - if len(fg_inds) > num_fg: - disable_inds = npr.choice(fg_inds, size=(len(fg_inds) - num_fg), replace=False) - if DEBUG: - disable_inds = fg_inds[:(len(fg_inds) - num_fg)] - labels[disable_inds] = -1 - - # subsample negative labels if we have too many - num_bg = config.TRAIN.RPN_BATCH_SIZE - np.sum(labels == 1) - bg_inds = np.where(labels == 0)[0] - if len(bg_inds) > num_bg: - disable_inds = npr.choice(bg_inds, size=(len(bg_inds) - num_bg), replace=False) - if DEBUG: - disable_inds = bg_inds[:(len(bg_inds) - num_bg)] - labels[disable_inds] = -1 - - #fg_inds = np.where(labels == 1)[0] - #num_fg = len(fg_inds) - #num_bg = num_fg*int(1.0/config.TRAIN.RPN_FG_FRACTION-1) - - #bg_inds = np.where(labels == 0)[0] - #if len(bg_inds) > num_bg: - # disable_inds = npr.choice(bg_inds, size=(len(bg_inds) - num_bg), replace=False) - # if DEBUG: - # disable_inds = bg_inds[:(len(bg_inds) - num_bg)] - # labels[disable_inds] = -1 - else: - fg_inds = np.where(labels == 1)[0] - num_fg = len(fg_inds) - bg_inds = np.where(labels == 0)[0] - num_bg = len(bg_inds) - - #print('anchor stat', num_fg, num_bg) - - - bbox_targets = np.zeros((len(inds_inside), bbox_pred_len), dtype=np.float32) - if gt_boxes.size > 0: - #print('GT', gt_boxes.shape, gt_boxes[argmax_overlaps, :4].shape) - bbox_targets[:,:] = bbox_transform(anchors, gt_boxes[argmax_overlaps, :]) - #bbox_targets[:,4] = gt_blur - - bbox_weights = np.zeros((len(inds_inside), bbox_pred_len), dtype=np.float32) - #bbox_weights[labels == 1, :] = np.array(config.TRAIN.RPN_BBOX_WEIGHTS) - bbox_weights[labels == 1, 0:4] = 1.0 - if bbox_pred_len>4: - bbox_weights[labels == 1, 4:bbox_pred_len] = 0.1 - - if landmark: - landmark_targets = np.zeros((len(inds_inside), landmark_pred_len), dtype=np.float32) - #landmark_weights = np.zeros((len(inds_inside), 10), dtype=np.float32) - landmark_weights = np.zeros((len(inds_inside), landmark_pred_len), dtype=np.float32) - #landmark_weights[labels == 1, :] = np.array(config.TRAIN.RPN_LANDMARK_WEIGHTS) - if landmark_pred_len==10: - landmark_weights[labels == 1, :] = 1.0 - elif landmark_pred_len==15: - v = [1.0, 1.0, 0.1] * 5 - assert len(v)==15 - landmark_weights[labels == 1, :] = np.array(v) - else: - assert False - #TODO here - if gt_landmarks.size > 0: - #print('AAA',argmax_overlaps) - a_landmarks = gt_landmarks[argmax_overlaps,:,:] - landmark_targets[:] = landmark_transform(anchors, a_landmarks) - invalid = np.where(a_landmarks[:,0,2]<0.0)[0] - #assert len(invalid)==0 - #landmark_weights[invalid, :] = np.array(config.TRAIN.RPN_INVALID_LANDMARK_WEIGHTS) - landmark_weights[invalid, :] = 0.0 - - #if DEBUG: - # _sums = bbox_targets[labels == 1, :].sum(axis=0) - # _squared_sums = (bbox_targets[labels == 1, :] ** 2).sum(axis=0) - # _counts = np.sum(labels == 1) - # means = _sums / (_counts + 1e-14) - # stds = np.sqrt(_squared_sums / _counts - means ** 2) - # print 'means', means - # print 'stdevs', stds - # map up to original set of anchors - #print(labels.shape, total_anchors, inds_inside.shape, inds_inside[0], inds_inside[-1]) - labels = _unmap(labels, total_anchors, inds_inside, fill=-1) - bbox_targets = _unmap(bbox_targets, total_anchors, inds_inside, fill=0) - bbox_weights = _unmap(bbox_weights, total_anchors, inds_inside, fill=0) - if landmark: - landmark_targets = _unmap(landmark_targets, total_anchors, inds_inside, fill=0) - landmark_weights = _unmap(landmark_weights, total_anchors, inds_inside, fill=0) - #print('CC', anchors.shape, len(inds_inside)) - - #if DEBUG: - # if gt_boxes.size > 0: - # print 'rpn: max max_overlaps', np.max(max_overlaps) - # print 'rpn: num_positives', np.sum(labels == 1) - # print 'rpn: num_negatives', np.sum(labels == 0) - # _fg_sum = np.sum(labels == 1) - # _bg_sum = np.sum(labels == 0) - # _count = 1 - # print 'rpn: num_positive avg', _fg_sum / _count - # print 'rpn: num_negative avg', _bg_sum / _count - - # resahpe - label_list = list() - bbox_target_list = list() - bbox_weight_list = list() - if landmark: - landmark_target_list = list() - landmark_weight_list = list() - anchors_num_range = [0] + anchors_num_list - label = {} - for i in range(len(feat_strides)): - stride = feat_strides[i] - feat_height, feat_width = feat_infos[i] - A = A_list[i] - _label = labels[sum(anchors_num_range[:i+1]):sum(anchors_num_range[:i+1])+anchors_num_range[i+1]] - if select_stride>0 and stride!=select_stride: - #print('set', stride, select_stride) - _label[:] = -1 - #print('_label', _label.shape, select_stride) - #_fg_inds = np.where(_label == 1)[0] - #n_fg = len(_fg_inds) - #STAT[0]+=1 - #STAT[stride]+=n_fg - #if STAT[0]%100==0: - # print('rpn_stat', STAT, file=sys.stderr) - bbox_target = bbox_targets[sum(anchors_num_range[:i+1]):sum(anchors_num_range[:i+1])+anchors_num_range[i+1]] - bbox_weight = bbox_weights[sum(anchors_num_range[:i+1]):sum(anchors_num_range[:i+1])+anchors_num_range[i+1]] - if landmark: - landmark_target = landmark_targets[sum(anchors_num_range[:i+1]):sum(anchors_num_range[:i+1])+anchors_num_range[i+1]] - landmark_weight = landmark_weights[sum(anchors_num_range[:i+1]):sum(anchors_num_range[:i+1])+anchors_num_range[i+1]] - - _label = _label.reshape((1, feat_height, feat_width, A)).transpose(0, 3, 1, 2) - _label = _label.reshape((1, A * feat_height * feat_width)) - bbox_target = bbox_target.reshape((1, feat_height*feat_width, A * bbox_pred_len)).transpose(0, 2, 1) - bbox_weight = bbox_weight.reshape((1, feat_height*feat_width, A * bbox_pred_len)).transpose((0, 2, 1)) - label['%s_label_stride%d'%(prefix, stride)] = _label - label['%s_bbox_target_stride%d'%(prefix,stride)] = bbox_target - label['%s_bbox_weight_stride%d'%(prefix,stride)] = bbox_weight - if landmark: - landmark_target = landmark_target.reshape((1, feat_height*feat_width, A * landmark_pred_len)).transpose(0, 2, 1) - landmark_weight = landmark_weight.reshape((1, feat_height*feat_width, A * landmark_pred_len)).transpose((0, 2, 1)) - label['%s_landmark_target_stride%d'%(prefix,stride)] = landmark_target - label['%s_landmark_weight_stride%d'%(prefix,stride)] = landmark_weight - #print('in_rpn', stride,_label.shape, bbox_target.shape, bbox_weight.shape, file=sys.stderr) - label_list.append(_label) - #print('DD', _label.shape) - bbox_target_list.append(bbox_target) - bbox_weight_list.append(bbox_weight) - if landmark: - landmark_target_list.append(landmark_target) - landmark_weight_list.append(landmark_weight) - - label_concat = np.concatenate(label_list, axis=1) - bbox_target_concat = np.concatenate(bbox_target_list, axis=2) - bbox_weight_concat = np.concatenate(bbox_weight_list, axis=2) - #fg_inds = np.where(label_concat[0] == 1)[0] - #print('fg_inds_in_rpn2', fg_inds, file=sys.stderr) - - label.update({'%s_label'%prefix: label_concat, - '%s_bbox_target'%prefix: bbox_target_concat, - '%s_bbox_weight'%prefix: bbox_weight_concat} - ) - if landmark: - landmark_target_concat = np.concatenate(landmark_target_list, axis=2) - landmark_weight_concat = np.concatenate(landmark_weight_list, axis=2) - label['%s_landmark_target'%prefix] = landmark_target_concat - label['%s_landmark_weight'%prefix] = landmark_weight_concat - return label - - -class AA: - def __init__(self, feat_shape): - self.feat_shape = feat_shape - feat_strides = config.RPN_FEAT_STRIDE - anchors_list = [] - anchors_num_list = [] - inds_inside_list = [] - feat_infos = [] - A_list = [] - DEBUG = False - for i in range(len(feat_strides)): - stride = feat_strides[i] - sstride = str(stride) - base_size = config.RPN_ANCHOR_CFG[sstride]['BASE_SIZE'] - allowed_border = config.RPN_ANCHOR_CFG[sstride]['ALLOWED_BORDER'] - ratios = config.RPN_ANCHOR_CFG[sstride]['RATIOS'] - scales = config.RPN_ANCHOR_CFG[sstride]['SCALES'] - base_anchors = generate_anchors(base_size=base_size, ratios=list(ratios), scales=np.array(scales, dtype=np.float32), stride = stride, dense_anchor = config.DENSE_ANCHOR) - num_anchors = base_anchors.shape[0] - feat_height, feat_width = feat_shape[i][-2:] - feat_stride = feat_strides[i] - feat_infos.append([feat_height, feat_width]) - - A = num_anchors - A_list.append(A) - K = feat_height * feat_width - - all_anchors = anchors_plane(feat_height, feat_width, feat_stride, base_anchors) - all_anchors = all_anchors.reshape((K * A, 4)) - #print('anchor0', stride, all_anchors[0]) - - total_anchors = int(K * A) - anchors_num_list.append(total_anchors) - # only keep anchors inside the image - inds_inside = np.where((all_anchors[:, 0] >= -allowed_border) & - (all_anchors[:, 1] >= -allowed_border) & - (all_anchors[:, 2] < config.SCALES[0][1] + allowed_border) & - (all_anchors[:, 3] < config.SCALES[0][1] + allowed_border))[0] - if DEBUG: - print('total_anchors', total_anchors) - print('inds_inside', len(inds_inside)) - - # keep only inside anchors - anchors = all_anchors[inds_inside, :] - #print('AA', anchors.shape, len(inds_inside)) - - anchors_list.append(anchors) - inds_inside_list.append(inds_inside) - anchors = np.concatenate(anchors_list) - for i in range(1, len(inds_inside_list)): - inds_inside_list[i] = inds_inside_list[i] + sum(anchors_num_list[:i]) - inds_inside = np.concatenate(inds_inside_list) - #self.anchors_list = anchors_list - #self.inds_inside_list = inds_inside_list - self.anchors = anchors - self.inds_inside = inds_inside - self.anchors_num_list = anchors_num_list - self.feat_infos = feat_infos - self.A_list = A_list - self._times = [0.0, 0.0, 0.0, 0.0] - - @staticmethod - def _unmap(data, count, inds, fill=0): - """" unmap a subset inds of data into original data of size count """ - if len(data.shape) == 1: - ret = np.empty((count,), dtype=np.float32) - ret.fill(fill) - ret[inds] = data - else: - ret = np.empty((count,) + data.shape[1:], dtype=np.float32) - ret.fill(fill) - ret[inds, :] = data - return ret - - def assign_anchor_fpn(self, gt_label, im_info, landmark=False, prefix='face', select_stride=0): - - #ta = datetime.datetime.now() - im_info = im_info[0] - gt_boxes = gt_label['gt_boxes'] - # clean up boxes - nonneg = np.where(gt_boxes[:, 4] != -1)[0] - gt_boxes = gt_boxes[nonneg] - if config.USE_BLUR: - gt_blur = gt_label['gt_blur'] - gt_blur = gt_blur[nonneg] - if landmark: - gt_landmarks = gt_label['gt_landmarks'] - gt_landmarks = gt_landmarks[nonneg] - assert gt_boxes.shape[0]==gt_landmarks.shape[0] - #scales = np.array(scales, dtype=np.float32) - feat_strides = config.RPN_FEAT_STRIDE - bbox_pred_len = 4 - landmark_pred_len = 10 - if config.USE_BLUR: - gt_boxes[:,4] = gt_blur - bbox_pred_len = 5 - if config.USE_OCCLUSION: - landmark_pred_len = 15 - - #anchors_list = self.anchors_list - #inds_inside_list = self.inds_inside_list - anchors = self.anchors - inds_inside = self.inds_inside - anchors_num_list = self.anchors_num_list - feat_infos = self.feat_infos - A_list = self.A_list - - total_anchors = sum(anchors_num_list) - #print('total_anchors', anchors.shape[0], len(inds_inside), file=sys.stderr) - - # label: 1 is positive, 0 is negative, -1 is dont care - labels = np.empty((len(inds_inside),), dtype=np.float32) - labels.fill(-1) - #print('BB', anchors.shape, len(inds_inside)) - #print('gt_boxes', gt_boxes.shape, file=sys.stderr) - #tb = datetime.datetime.now() - #self._times[0] += (tb-ta).total_seconds() - #ta = datetime.datetime.now() - - if gt_boxes.size > 0: - # overlap between the anchors and the gt boxes - # overlaps (ex, gt) - overlaps = bbox_overlaps(anchors.astype(np.float), gt_boxes.astype(np.float)) - argmax_overlaps = overlaps.argmax(axis=1) - #print('AAA', argmax_overlaps.shape) - max_overlaps = overlaps[np.arange(len(inds_inside)), argmax_overlaps] - gt_argmax_overlaps = overlaps.argmax(axis=0) - gt_max_overlaps = overlaps[gt_argmax_overlaps, np.arange(overlaps.shape[1])] - gt_argmax_overlaps = np.where(overlaps == gt_max_overlaps)[0] - - if not config.TRAIN.RPN_CLOBBER_POSITIVES: - # assign bg labels first so that positive labels can clobber them - labels[max_overlaps < config.TRAIN.RPN_NEGATIVE_OVERLAP] = 0 - - # fg label: for each gt, anchor with highest overlap - if config.TRAIN.RPN_FORCE_POSITIVE: - labels[gt_argmax_overlaps] = 1 - - # fg label: above threshold IoU - labels[max_overlaps >= config.TRAIN.RPN_POSITIVE_OVERLAP] = 1 - - if config.TRAIN.RPN_CLOBBER_POSITIVES: - # assign bg labels last so that negative labels can clobber positives - labels[max_overlaps < config.TRAIN.RPN_NEGATIVE_OVERLAP] = 0 - else: - labels[:] = 0 - fg_inds = np.where(labels == 1)[0] - #print('fg count', len(fg_inds)) - - # subsample positive labels if we have too many - if config.TRAIN.RPN_ENABLE_OHEM==0: - fg_inds = np.where(labels == 1)[0] - num_fg = int(config.TRAIN.RPN_FG_FRACTION * config.TRAIN.RPN_BATCH_SIZE) - if len(fg_inds) > num_fg: - disable_inds = npr.choice(fg_inds, size=(len(fg_inds) - num_fg), replace=False) - if DEBUG: - disable_inds = fg_inds[:(len(fg_inds) - num_fg)] - labels[disable_inds] = -1 - - # subsample negative labels if we have too many - num_bg = config.TRAIN.RPN_BATCH_SIZE - np.sum(labels == 1) - bg_inds = np.where(labels == 0)[0] - if len(bg_inds) > num_bg: - disable_inds = npr.choice(bg_inds, size=(len(bg_inds) - num_bg), replace=False) - if DEBUG: - disable_inds = bg_inds[:(len(bg_inds) - num_bg)] - labels[disable_inds] = -1 - - #fg_inds = np.where(labels == 1)[0] - #num_fg = len(fg_inds) - #num_bg = num_fg*int(1.0/config.TRAIN.RPN_FG_FRACTION-1) - - #bg_inds = np.where(labels == 0)[0] - #if len(bg_inds) > num_bg: - # disable_inds = npr.choice(bg_inds, size=(len(bg_inds) - num_bg), replace=False) - # if DEBUG: - # disable_inds = bg_inds[:(len(bg_inds) - num_bg)] - # labels[disable_inds] = -1 - else: - fg_inds = np.where(labels == 1)[0] - num_fg = len(fg_inds) - bg_inds = np.where(labels == 0)[0] - num_bg = len(bg_inds) - - #print('anchor stat', num_fg, num_bg) - - - bbox_targets = np.zeros((len(inds_inside), bbox_pred_len), dtype=np.float32) - if gt_boxes.size > 0: - #print('GT', gt_boxes.shape, gt_boxes[argmax_overlaps, :4].shape) - bbox_targets[:,:] = bbox_transform(anchors, gt_boxes[argmax_overlaps, :]) - #bbox_targets[:,4] = gt_blur - #tb = datetime.datetime.now() - #self._times[1] += (tb-ta).total_seconds() - #ta = datetime.datetime.now() - - bbox_weights = np.zeros((len(inds_inside), bbox_pred_len), dtype=np.float32) - #bbox_weights[labels == 1, :] = np.array(config.TRAIN.RPN_BBOX_WEIGHTS) - bbox_weights[labels == 1, 0:4] = 1.0 - if bbox_pred_len>4: - bbox_weights[labels == 1, 4:bbox_pred_len] = 0.1 - - if landmark: - landmark_targets = np.zeros((len(inds_inside), landmark_pred_len), dtype=np.float32) - #landmark_weights = np.zeros((len(inds_inside), 10), dtype=np.float32) - landmark_weights = np.zeros((len(inds_inside), landmark_pred_len), dtype=np.float32) - #landmark_weights[labels == 1, :] = np.array(config.TRAIN.RPN_LANDMARK_WEIGHTS) - if landmark_pred_len==10: - landmark_weights[labels == 1, :] = 1.0 - elif landmark_pred_len==15: - v = [1.0, 1.0, 0.1] * 5 - assert len(v)==15 - landmark_weights[labels == 1, :] = np.array(v) - else: - assert False - #TODO here - if gt_landmarks.size > 0: - #print('AAA',argmax_overlaps) - a_landmarks = gt_landmarks[argmax_overlaps,:,:] - landmark_targets[:] = landmark_transform(anchors, a_landmarks) - invalid = np.where(a_landmarks[:,0,2]<0.0)[0] - #assert len(invalid)==0 - #landmark_weights[invalid, :] = np.array(config.TRAIN.RPN_INVALID_LANDMARK_WEIGHTS) - landmark_weights[invalid, :] = 0.0 - #tb = datetime.datetime.now() - #self._times[2] += (tb-ta).total_seconds() - #ta = datetime.datetime.now() - - #if DEBUG: - # _sums = bbox_targets[labels == 1, :].sum(axis=0) - # _squared_sums = (bbox_targets[labels == 1, :] ** 2).sum(axis=0) - # _counts = np.sum(labels == 1) - # means = _sums / (_counts + 1e-14) - # stds = np.sqrt(_squared_sums / _counts - means ** 2) - # print 'means', means - # print 'stdevs', stds - # map up to original set of anchors - #print(labels.shape, total_anchors, inds_inside.shape, inds_inside[0], inds_inside[-1]) - labels = AA._unmap(labels, total_anchors, inds_inside, fill=-1) - bbox_targets = AA._unmap(bbox_targets, total_anchors, inds_inside, fill=0) - bbox_weights = AA._unmap(bbox_weights, total_anchors, inds_inside, fill=0) - if landmark: - landmark_targets = AA._unmap(landmark_targets, total_anchors, inds_inside, fill=0) - landmark_weights = AA._unmap(landmark_weights, total_anchors, inds_inside, fill=0) - #print('CC', anchors.shape, len(inds_inside)) - - bbox_targets[:, 0::4] = bbox_targets[:,0::4] / config.TRAIN.BBOX_STDS[0] - bbox_targets[:, 1::4] = bbox_targets[:,1::4] / config.TRAIN.BBOX_STDS[1] - bbox_targets[:, 2::4] = bbox_targets[:,2::4] / config.TRAIN.BBOX_STDS[2] - bbox_targets[:, 3::4] = bbox_targets[:,3::4] / config.TRAIN.BBOX_STDS[3] - landmark_targets /= config.TRAIN.LANDMARK_STD - #print('applied STD') - - #if DEBUG: - # if gt_boxes.size > 0: - # print 'rpn: max max_overlaps', np.max(max_overlaps) - # print 'rpn: num_positives', np.sum(labels == 1) - # print 'rpn: num_negatives', np.sum(labels == 0) - # _fg_sum = np.sum(labels == 1) - # _bg_sum = np.sum(labels == 0) - # _count = 1 - # print 'rpn: num_positive avg', _fg_sum / _count - # print 'rpn: num_negative avg', _bg_sum / _count - - # resahpe - label_list = list() - bbox_target_list = list() - bbox_weight_list = list() - if landmark: - landmark_target_list = list() - landmark_weight_list = list() - anchors_num_range = [0] + anchors_num_list - label = {} - for i in range(len(feat_strides)): - stride = feat_strides[i] - feat_height, feat_width = feat_infos[i] - A = A_list[i] - _label = labels[sum(anchors_num_range[:i+1]):sum(anchors_num_range[:i+1])+anchors_num_range[i+1]] - if select_stride>0 and stride!=select_stride: - #print('set', stride, select_stride) - _label[:] = -1 - #print('_label', _label.shape, select_stride) - #_fg_inds = np.where(_label == 1)[0] - #n_fg = len(_fg_inds) - #STAT[0]+=1 - #STAT[stride]+=n_fg - #if STAT[0]%100==0: - # print('rpn_stat', STAT, file=sys.stderr) - bbox_target = bbox_targets[sum(anchors_num_range[:i+1]):sum(anchors_num_range[:i+1])+anchors_num_range[i+1]] - bbox_weight = bbox_weights[sum(anchors_num_range[:i+1]):sum(anchors_num_range[:i+1])+anchors_num_range[i+1]] - if landmark: - landmark_target = landmark_targets[sum(anchors_num_range[:i+1]):sum(anchors_num_range[:i+1])+anchors_num_range[i+1]] - landmark_weight = landmark_weights[sum(anchors_num_range[:i+1]):sum(anchors_num_range[:i+1])+anchors_num_range[i+1]] - - _label = _label.reshape((1, feat_height, feat_width, A)).transpose(0, 3, 1, 2) - _label = _label.reshape((1, A * feat_height * feat_width)) - bbox_target = bbox_target.reshape((1, feat_height*feat_width, A * bbox_pred_len)).transpose(0, 2, 1) - bbox_weight = bbox_weight.reshape((1, feat_height*feat_width, A * bbox_pred_len)).transpose((0, 2, 1)) - label['%s_label_stride%d'%(prefix, stride)] = _label - label['%s_bbox_target_stride%d'%(prefix,stride)] = bbox_target - label['%s_bbox_weight_stride%d'%(prefix,stride)] = bbox_weight - if landmark: - landmark_target = landmark_target.reshape((1, feat_height*feat_width, A * landmark_pred_len)).transpose(0, 2, 1) - landmark_weight = landmark_weight.reshape((1, feat_height*feat_width, A * landmark_pred_len)).transpose((0, 2, 1)) - label['%s_landmark_target_stride%d'%(prefix,stride)] = landmark_target - label['%s_landmark_weight_stride%d'%(prefix,stride)] = landmark_weight - #print('in_rpn', stride,_label.shape, bbox_target.shape, bbox_weight.shape, file=sys.stderr) - label_list.append(_label) - #print('DD', _label.shape) - bbox_target_list.append(bbox_target) - bbox_weight_list.append(bbox_weight) - if landmark: - landmark_target_list.append(landmark_target) - landmark_weight_list.append(landmark_weight) - - label_concat = np.concatenate(label_list, axis=1) - bbox_target_concat = np.concatenate(bbox_target_list, axis=2) - bbox_weight_concat = np.concatenate(bbox_weight_list, axis=2) - #fg_inds = np.where(label_concat[0] == 1)[0] - #print('fg_inds_in_rpn2', fg_inds, file=sys.stderr) - - label.update({'%s_label'%prefix: label_concat, - '%s_bbox_target'%prefix: bbox_target_concat, - '%s_bbox_weight'%prefix: bbox_weight_concat} - ) - if landmark: - landmark_target_concat = np.concatenate(landmark_target_list, axis=2) - landmark_weight_concat = np.concatenate(landmark_weight_list, axis=2) - label['%s_landmark_target'%prefix] = landmark_target_concat - label['%s_landmark_weight'%prefix] = landmark_weight_concat - #tb = datetime.datetime.now() - #self._times[3] += (tb-ta).total_seconds() - #ta = datetime.datetime.now() - #print(self._times) - return label diff --git a/RetinaFace/rcnn/symbol/pyramidbox.py b/RetinaFace/rcnn/symbol/pyramidbox.py deleted file mode 100644 index dae2537..0000000 --- a/RetinaFace/rcnn/symbol/pyramidbox.py +++ /dev/null @@ -1,427 +0,0 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -import six -import paddle.fluid as fluid -from paddle.fluid.param_attr import ParamAttr -from paddle.fluid.initializer import Xavier -from paddle.fluid.initializer import Constant -from paddle.fluid.initializer import Bilinear -from paddle.fluid.regularizer import L2Decay - - -def conv_bn(input, filter, ksize, stride, padding, act='relu', bias_attr=False): - conv = fluid.layers.conv2d( - input=input, - filter_size=ksize, - num_filters=filter, - stride=stride, - padding=padding, - act=None, - bias_attr=bias_attr) - return fluid.layers.batch_norm(input=conv, act=act) - - -def conv_block(input, groups, filters, ksizes, strides=None, with_pool=True): - assert len(filters) == groups - assert len(ksizes) == groups - strides = [1] * groups if strides is None else strides - w_attr = ParamAttr(learning_rate=1., initializer=Xavier()) - b_attr = ParamAttr(learning_rate=2., regularizer=L2Decay(0.)) - conv = input - for i in six.moves.xrange(groups): - conv = fluid.layers.conv2d( - input=conv, - num_filters=filters[i], - filter_size=ksizes[i], - stride=strides[i], - padding=(ksizes[i] - 1) // 2, - param_attr=w_attr, - bias_attr=b_attr, - act='relu') - if with_pool: - pool = fluid.layers.pool2d( - input=conv, - pool_size=2, - pool_type='max', - pool_stride=2, - ceil_mode=True) - return conv, pool - else: - return conv - - -class PyramidBox(object): - def __init__(self, - data_shape, - num_classes=None, - use_transposed_conv2d=True, - is_infer=False, - sub_network=False): - """ - TODO(qingqing): add comments. - """ - self.data_shape = data_shape - self.min_sizes = [16., 32., 64., 128., 256., 512.] - self.steps = [4., 8., 16., 32., 64., 128.] - self.num_classes = num_classes - self.use_transposed_conv2d = use_transposed_conv2d - self.is_infer = is_infer - self.sub_network = sub_network - - # the base network is VGG with atrous layers - self._input() - self._vgg() - if sub_network: - self._low_level_fpn() - self._cpm_module() - self._pyramidbox() - else: - self._vgg_ssd() - - def feeds(self): - if self.is_infer: - return [self.image] - else: - return [self.image, self.face_box, self.head_box, self.gt_label] - - def _input(self): - self.image = fluid.layers.data( - name='image', shape=self.data_shape, dtype='float32') - if not self.is_infer: - self.face_box = fluid.layers.data( - name='face_box', shape=[4], dtype='float32', lod_level=1) - self.head_box = fluid.layers.data( - name='head_box', shape=[4], dtype='float32', lod_level=1) - self.gt_label = fluid.layers.data( - name='gt_label', shape=[1], dtype='int32', lod_level=1) - - def _vgg(self): - self.conv1, self.pool1 = conv_block(self.image, 2, [64] * 2, [3] * 2) - self.conv2, self.pool2 = conv_block(self.pool1, 2, [128] * 2, [3] * 2) - - #priorbox min_size is 16 - self.conv3, self.pool3 = conv_block(self.pool2, 3, [256] * 3, [3] * 3) - #priorbox min_size is 32 - self.conv4, self.pool4 = conv_block(self.pool3, 3, [512] * 3, [3] * 3) - #priorbox min_size is 64 - self.conv5, self.pool5 = conv_block(self.pool4, 3, [512] * 3, [3] * 3) - - # fc6 and fc7 in paper, priorbox min_size is 128 - self.conv6 = conv_block( - self.pool5, 2, [1024, 1024], [3, 1], with_pool=False) - # conv6_1 and conv6_2 in paper, priorbox min_size is 256 - self.conv7 = conv_block( - self.conv6, 2, [256, 512], [1, 3], [1, 2], with_pool=False) - # conv7_1 and conv7_2 in paper, priorbox mini_size is 512 - self.conv8 = conv_block( - self.conv7, 2, [128, 256], [1, 3], [1, 2], with_pool=False) - - def _low_level_fpn(self): - """ - Low-level feature pyramid network. - """ - - def fpn(up_from, up_to): - ch = up_to.shape[1] - b_attr = ParamAttr(learning_rate=2., regularizer=L2Decay(0.)) - conv1 = fluid.layers.conv2d( - up_from, ch, 1, act='relu', bias_attr=b_attr) - if self.use_transposed_conv2d: - w_attr = ParamAttr( - learning_rate=0., - regularizer=L2Decay(0.), - initializer=Bilinear()) - upsampling = fluid.layers.conv2d_transpose( - conv1, - ch, - output_size=None, - filter_size=4, - padding=1, - stride=2, - groups=ch, - param_attr=w_attr, - bias_attr=False, - use_cudnn=True) - else: - upsampling = fluid.layers.resize_bilinear( - conv1, out_shape=up_to.shape[2:]) - - conv2 = fluid.layers.conv2d( - up_to, ch, 1, act='relu', bias_attr=b_attr) - if self.is_infer: - upsampling = fluid.layers.crop(upsampling, shape=conv2) - # eltwise mul - conv_fuse = upsampling * conv2 - return conv_fuse - - self.lfpn2_on_conv5 = fpn(self.conv6, self.conv5) - self.lfpn1_on_conv4 = fpn(self.lfpn2_on_conv5, self.conv4) - self.lfpn0_on_conv3 = fpn(self.lfpn1_on_conv4, self.conv3) - - def _cpm_module(self): - """ - Context-sensitive Prediction Module - """ - - def cpm(input): - # residual - branch1 = conv_bn(input, 1024, 1, 1, 0, None) - branch2a = conv_bn(input, 256, 1, 1, 0, act='relu') - branch2b = conv_bn(branch2a, 256, 3, 1, 1, act='relu') - branch2c = conv_bn(branch2b, 1024, 1, 1, 0, None) - sum = branch1 + branch2c - rescomb = fluid.layers.relu(x=sum) - - # ssh - b_attr = ParamAttr(learning_rate=2., regularizer=L2Decay(0.)) - ssh_1 = fluid.layers.conv2d(rescomb, 256, 3, 1, 1, bias_attr=b_attr) - ssh_dimred = fluid.layers.conv2d( - rescomb, 128, 3, 1, 1, act='relu', bias_attr=b_attr) - ssh_2 = fluid.layers.conv2d( - ssh_dimred, 128, 3, 1, 1, bias_attr=b_attr) - ssh_3a = fluid.layers.conv2d( - ssh_dimred, 128, 3, 1, 1, act='relu', bias_attr=b_attr) - ssh_3b = fluid.layers.conv2d(ssh_3a, 128, 3, 1, 1, bias_attr=b_attr) - - ssh_concat = fluid.layers.concat([ssh_1, ssh_2, ssh_3b], axis=1) - ssh_out = fluid.layers.relu(x=ssh_concat) - return ssh_out - - self.ssh_conv3 = cpm(self.lfpn0_on_conv3) - self.ssh_conv4 = cpm(self.lfpn1_on_conv4) - self.ssh_conv5 = cpm(self.lfpn2_on_conv5) - self.ssh_conv6 = cpm(self.conv6) - self.ssh_conv7 = cpm(self.conv7) - self.ssh_conv8 = cpm(self.conv8) - - def _l2_norm_scale(self, input, init_scale=1.0, channel_shared=False): - from paddle.fluid.layer_helper import LayerHelper - helper = LayerHelper("Scale") - l2_norm = fluid.layers.l2_normalize( - input, axis=1) # l2 norm along channel - shape = [1] if channel_shared else [input.shape[1]] - scale = helper.create_parameter( - attr=helper.param_attr, - shape=shape, - dtype=input.dtype, - default_initializer=Constant(init_scale)) - out = fluid.layers.elementwise_mul( - x=l2_norm, y=scale, axis=-1 if channel_shared else 1) - return out - - def _pyramidbox(self): - """ - Get prior-boxes and pyramid-box - """ - self.ssh_conv3_norm = self._l2_norm_scale( - self.ssh_conv3, init_scale=10.) - self.ssh_conv4_norm = self._l2_norm_scale(self.ssh_conv4, init_scale=8.) - self.ssh_conv5_norm = self._l2_norm_scale(self.ssh_conv5, init_scale=5.) - - def permute_and_reshape(input, last_dim): - trans = fluid.layers.transpose(input, perm=[0, 2, 3, 1]) - compile_shape = [ - trans.shape[0], np.prod(trans.shape[1:]) // last_dim, last_dim - ] - run_shape = fluid.layers.assign( - np.array([0, -1, last_dim]).astype("int32")) - return fluid.layers.reshape( - trans, shape=compile_shape, actual_shape=run_shape) - - face_locs, face_confs = [], [] - head_locs, head_confs = [], [] - boxes, vars = [], [] - inputs = [ - self.ssh_conv3_norm, self.ssh_conv4_norm, self.ssh_conv5_norm, - self.ssh_conv6, self.ssh_conv7, self.ssh_conv8 - ] - b_attr = ParamAttr(learning_rate=2., regularizer=L2Decay(0.)) - for i, input in enumerate(inputs): - mbox_loc = fluid.layers.conv2d(input, 8, 3, 1, 1, bias_attr=b_attr) - face_loc, head_loc = fluid.layers.split( - mbox_loc, num_or_sections=2, dim=1) - face_loc = permute_and_reshape(face_loc, 4) - head_loc = permute_and_reshape(head_loc, 4) - - mbox_conf = fluid.layers.conv2d(input, 6, 3, 1, 1, bias_attr=b_attr) - face_conf1, face_conf3, head_conf = fluid.layers.split( - mbox_conf, num_or_sections=[1, 3, 2], dim=1) - face_conf3_maxin = fluid.layers.reduce_max( - face_conf3, dim=1, keep_dim=True) - face_conf = fluid.layers.concat( - [face_conf1, face_conf3_maxin], axis=1) - - face_conf = permute_and_reshape(face_conf, 2) - head_conf = permute_and_reshape(head_conf, 2) - - face_locs.append(face_loc) - face_confs.append(face_conf) - - head_locs.append(head_loc) - head_confs.append(head_conf) - - box, var = fluid.layers.prior_box( - input, - self.image, - min_sizes=[self.min_sizes[i]], - steps=[self.steps[i]] * 2, - aspect_ratios=[1.], - clip=False, - flip=True, - offset=0.5) - box = fluid.layers.reshape(box, shape=[-1, 4]) - var = fluid.layers.reshape(var, shape=[-1, 4]) - - boxes.append(box) - vars.append(var) - - self.face_mbox_loc = fluid.layers.concat(face_locs, axis=1) - self.face_mbox_conf = fluid.layers.concat(face_confs, axis=1) - - self.head_mbox_loc = fluid.layers.concat(head_locs, axis=1) - self.head_mbox_conf = fluid.layers.concat(head_confs, axis=1) - - self.prior_boxes = fluid.layers.concat(boxes) - self.box_vars = fluid.layers.concat(vars) - - def _vgg_ssd(self): - self.conv3_norm = self._l2_norm_scale(self.conv3, init_scale=10.) - self.conv4_norm = self._l2_norm_scale(self.conv4, init_scale=8.) - self.conv5_norm = self._l2_norm_scale(self.conv5, init_scale=5.) - - def permute_and_reshape(input, last_dim): - trans = fluid.layers.transpose(input, perm=[0, 2, 3, 1]) - compile_shape = [ - trans.shape[0], np.prod(trans.shape[1:]) // last_dim, last_dim - ] - run_shape = fluid.layers.assign( - np.array([0, -1, last_dim]).astype("int32")) - return fluid.layers.reshape( - trans, shape=compile_shape, actual_shape=run_shape) - - locs, confs = [], [] - boxes, vars = [], [] - b_attr = ParamAttr(learning_rate=2., regularizer=L2Decay(0.)) - - # conv3 - mbox_loc = fluid.layers.conv2d( - self.conv3_norm, 4, 3, 1, 1, bias_attr=b_attr) - loc = permute_and_reshape(mbox_loc, 4) - mbox_conf = fluid.layers.conv2d( - self.conv3_norm, 4, 3, 1, 1, bias_attr=b_attr) - conf1, conf3 = fluid.layers.split( - mbox_conf, num_or_sections=[1, 3], dim=1) - conf3_maxin = fluid.layers.reduce_max(conf3, dim=1, keep_dim=True) - conf = fluid.layers.concat([conf1, conf3_maxin], axis=1) - conf = permute_and_reshape(conf, 2) - box, var = fluid.layers.prior_box( - self.conv3_norm, - self.image, - min_sizes=[16.], - steps=[4, 4], - aspect_ratios=[1.], - clip=False, - flip=True, - offset=0.5) - box = fluid.layers.reshape(box, shape=[-1, 4]) - var = fluid.layers.reshape(var, shape=[-1, 4]) - - locs.append(loc) - confs.append(conf) - boxes.append(box) - vars.append(var) - - min_sizes = [32., 64., 128., 256., 512.] - steps = [8., 16., 32., 64., 128.] - inputs = [ - self.conv4_norm, self.conv5_norm, self.conv6, self.conv7, self.conv8 - ] - for i, input in enumerate(inputs): - mbox_loc = fluid.layers.conv2d(input, 4, 3, 1, 1, bias_attr=b_attr) - loc = permute_and_reshape(mbox_loc, 4) - - mbox_conf = fluid.layers.conv2d(input, 2, 3, 1, 1, bias_attr=b_attr) - conf = permute_and_reshape(mbox_conf, 2) - box, var = fluid.layers.prior_box( - input, - self.image, - min_sizes=[min_sizes[i]], - steps=[steps[i]] * 2, - aspect_ratios=[1.], - clip=False, - flip=True, - offset=0.5) - box = fluid.layers.reshape(box, shape=[-1, 4]) - var = fluid.layers.reshape(var, shape=[-1, 4]) - - locs.append(loc) - confs.append(conf) - boxes.append(box) - vars.append(var) - - self.face_mbox_loc = fluid.layers.concat(locs, axis=1) - self.face_mbox_conf = fluid.layers.concat(confs, axis=1) - self.prior_boxes = fluid.layers.concat(boxes) - self.box_vars = fluid.layers.concat(vars) - - def vgg_ssd_loss(self): - loss = fluid.layers.ssd_loss( - self.face_mbox_loc, - self.face_mbox_conf, - self.face_box, - self.gt_label, - self.prior_boxes, - self.box_vars, - overlap_threshold=0.35, - neg_overlap=0.35) - loss = fluid.layers.reduce_sum(loss) - return loss - - def train(self): - face_loss = fluid.layers.ssd_loss( - self.face_mbox_loc, - self.face_mbox_conf, - self.face_box, - self.gt_label, - self.prior_boxes, - self.box_vars, - overlap_threshold=0.35, - neg_overlap=0.35) - face_loss.persistable = True - head_loss = fluid.layers.ssd_loss( - self.head_mbox_loc, - self.head_mbox_conf, - self.head_box, - self.gt_label, - self.prior_boxes, - self.box_vars, - overlap_threshold=0.35, - neg_overlap=0.35) - head_loss.persistable = True - face_loss = fluid.layers.reduce_sum(face_loss) - face_loss.persistable = True - head_loss = fluid.layers.reduce_sum(head_loss) - head_loss.persistable = True - total_loss = face_loss + head_loss - total_loss.persistable = True - return face_loss, head_loss, total_loss - - def infer(self, main_program=None): - if main_program is None: - test_program = fluid.default_main_program().clone(for_test=True) - else: - test_program = main_program.clone(for_test=True) - with fluid.program_guard(test_program): - face_nmsed_out = fluid.layers.detection_output( - self.face_mbox_loc, - self.face_mbox_conf, - self.prior_boxes, - self.box_vars, - nms_threshold=0.3, - nms_top_k=5000, - keep_top_k=750, - score_threshold=0.01) - return test_program, face_nmsed_out diff --git a/RetinaFace/rcnn/symbol/symbol_common.py b/RetinaFace/rcnn/symbol/symbol_common.py deleted file mode 100644 index 4db2618..0000000 --- a/RetinaFace/rcnn/symbol/symbol_common.py +++ /dev/null @@ -1,472 +0,0 @@ -import mxnet as mx -import mxnet.ndarray as nd -import numpy as np -from rcnn.config import config -from rcnn.PY_OP import rpn_fpn_ohem3, cascade_refine - -PREFIX = 'RF' -F1 = 0 -F2 = 0 -_bwm = 1.0 - -def conv_only(from_layer, name, num_filter, kernel=(1,1), pad=(0,0), \ - stride=(1,1), bias_wd_mult=0.0, shared_weight=None, shared_bias = None): - if shared_weight is None: - weight = mx.symbol.Variable(name="{}_weight".format(name), - init=mx.init.Normal(0.01), attr={'__lr_mult__': '1.0'}) - bias = mx.symbol.Variable(name="{}_bias".format(name), - init=mx.init.Constant(0.0), attr={'__lr_mult__': '2.0', '__wd_mult__': str(bias_wd_mult)}) - else: - weight = shared_weight - bias = shared_bias - print('reuse shared var in', name) - conv = mx.symbol.Convolution(data=from_layer, kernel=kernel, pad=pad, \ - stride=stride, num_filter=num_filter, name="{}".format(name), weight = weight, bias=bias) - return conv - -def conv_deformable(net, num_filter, num_group=1, act_type='relu',name=''): - if config.USE_DCN==1: - f = num_group*18 - conv_offset = mx.symbol.Convolution(name=name+'_conv_offset', data = net, - num_filter=f, pad=(1, 1), kernel=(3, 3), stride=(1, 1)) - net = mx.contrib.symbol.DeformableConvolution(name=name+"_conv", data=net, offset=conv_offset, - num_filter=num_filter, pad=(1,1), kernel=(3, 3), num_deformable_group=num_group, stride=(1, 1), no_bias=False) - else: - print('use dcnv2 at', name) - lr_mult = 0.1 - weight_var = mx.sym.Variable(name=name+'_conv2_offset_weight', init=mx.init.Zero(), lr_mult=lr_mult) - bias_var = mx.sym.Variable(name=name+'_conv2_offset_bias', init=mx.init.Zero(), lr_mult=lr_mult) - conv2_offset = mx.symbol.Convolution(name=name + '_conv2_offset', data=net, num_filter=27, - pad=(1, 1), kernel=(3, 3), stride=(1,1), weight=weight_var, bias=bias_var, lr_mult=lr_mult) - conv2_offset_t = mx.sym.slice_axis(conv2_offset, axis=1, begin=0, end=18) - conv2_mask = mx.sym.slice_axis(conv2_offset, axis=1, begin=18, end=None) - conv2_mask = 2 * mx.sym.Activation(conv2_mask, act_type='sigmoid') - - conv2 = mx.contrib.symbol.ModulatedDeformableConvolution(name=name + '_conv2', data=net, offset=conv2_offset_t, mask=conv2_mask, - num_filter=num_filter, pad=(1, 1), kernel=(3, 3), stride=(1,1), - num_deformable_group=num_group, no_bias=True) - net = conv2 - net = mx.sym.BatchNorm(data=net, fix_gamma=False, eps=2e-5, momentum=0.9, name=name + '_bn') - if len(act_type)>0: - net = mx.symbol.Activation(data=net, act_type=act_type, name=name+'_act') - return net - -def conv_act_layer_dw(from_layer, name, num_filter, kernel=(1,1), pad=(0,0), \ - stride=(1,1), act_type="relu", bias_wd_mult=0.0): - assert kernel[0]==3 - weight = mx.symbol.Variable(name="{}_weight".format(name), - init=mx.init.Normal(0.01), attr={'__lr_mult__': '1.0'}) - bias = mx.symbol.Variable(name="{}_bias".format(name), - init=mx.init.Constant(0.0), attr={'__lr_mult__': '2.0', '__wd_mult__': str(bias_wd_mult)}) - conv = mx.symbol.Convolution(data=from_layer, kernel=kernel, pad=pad, \ - stride=stride, num_filter=num_filter, num_group=num_filter, name="{}".format(name), weight=weight, bias=bias) - conv = mx.sym.BatchNorm(data=conv, fix_gamma=False, eps=2e-5, momentum=0.9, name=name + '_bn') - if len(act_type)>0: - relu = mx.symbol.Activation(data=conv, act_type=act_type, \ - name="{}_{}".format(name, act_type)) - else: - relu = conv - return relu - -def conv_act_layer(from_layer, name, num_filter, kernel=(1,1), pad=(0,0), \ - stride=(1,1), act_type="relu", bias_wd_mult=0.0, separable=False, filter_in = -1): - - if config.USE_DCN>1 and kernel==(3,3) and pad==(1,1) and stride==(1,1) and not separable: - return conv_deformable(from_layer, num_filter, num_group=1, act_type = act_type, name=name) - - if separable: - assert kernel[0]>1 - assert filter_in>0 - if not separable: - weight = mx.symbol.Variable(name="{}_weight".format(name), - init=mx.init.Normal(0.01), attr={'__lr_mult__': '1.0'}) - bias = mx.symbol.Variable(name="{}_bias".format(name), - init=mx.init.Constant(0.0), attr={'__lr_mult__': '2.0', '__wd_mult__': str(bias_wd_mult)}) - conv = mx.symbol.Convolution(data=from_layer, kernel=kernel, pad=pad, \ - stride=stride, num_filter=num_filter, name="{}".format(name), weight=weight, bias=bias) - conv = mx.sym.BatchNorm(data=conv, fix_gamma=False, eps=2e-5, momentum=0.9, name=name + '_bn') - else: - if filter_in<0: - filter_in = num_filter - conv = mx.symbol.Convolution(data=from_layer, kernel=kernel, pad=pad, \ - stride=stride, num_filter=filter_in, num_group=filter_in, name="{}_sep".format(name)) - conv = mx.sym.BatchNorm(data=conv, fix_gamma=False, eps=2e-5, momentum=0.9, name=name + '_sep_bn') - conv = mx.symbol.Activation(data=conv, act_type='relu', \ - name="{}_sep_bn_relu".format(name)) - conv = mx.symbol.Convolution(data=conv, kernel=(1,1), pad=(0,0), \ - stride=(1,1), num_filter=num_filter, name="{}".format(name)) - conv = mx.sym.BatchNorm(data=conv, fix_gamma=False, eps=2e-5, momentum=0.9, name=name + '_bn') - if len(act_type)>0: - relu = mx.symbol.Activation(data=conv, act_type=act_type, \ - name="{}_{}".format(name, act_type)) - else: - relu = conv - return relu - -def ssh_context_module(body, num_filter, filter_in, name): - conv_dimred = conv_act_layer(body, name+'_conv1', - num_filter, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='relu', separable=False, filter_in = filter_in) - conv5x5 = conv_act_layer(conv_dimred, name+'_conv2', - num_filter, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='', separable=False) - conv7x7_1 = conv_act_layer(conv_dimred, name+'_conv3_1', - num_filter, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='relu', separable=False) - conv7x7 = conv_act_layer(conv7x7_1, name+'_conv3_2', - num_filter, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='', separable=False) - return (conv5x5, conv7x7) - - -def ssh_detection_module(body, num_filter, filter_in, name): - assert num_filter%4==0 - conv3x3 = conv_act_layer(body, name+'_conv1', - num_filter//2, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='', separable=False, filter_in=filter_in) - #_filter = max(num_filter//4, 16) - _filter = num_filter//4 - conv5x5, conv7x7 = ssh_context_module(body, _filter, filter_in, name+'_context') - ret = mx.sym.concat(*[conv3x3, conv5x5, conv7x7], dim=1, name = name+'_concat') - ret = mx.symbol.Activation(data=ret, act_type='relu', name=name+'_concat_relu') - out_filter = num_filter//2+_filter*2 - if config.USE_DCN>0: - ret = conv_deformable(ret, num_filter = out_filter, name = name+'_concat_dcn') - return ret - -#def retina_context_module(body, kernel, num_filter, filter_in, name): -# conv_dimred = conv_act_layer(body, name+'_conv0', -# num_filter, kernel=(1,1), pad=(0,0), stride=(1, 1), act_type='relu', separable=False, filter_in = filter_in) -# conv1 = conv_act_layer(conv_dimred, name+'_conv1', -# num_filter*6, kernel=(1,1), pad=(0,0), stride=(1, 1), act_type='relu', separable=False, filter_in = filter_in) -# conv2 = conv_act_layer(conv1, name+'_conv2', -# num_filter*6, kernel=kernel, pad=((kernel[0]-1)//2, (kernel[1]-1)//2), stride=(1, 1), act_type='relu', separable=True, filter_in = num_filter*6) -# conv3 = conv_act_layer(conv2, name+'_conv3', -# num_filter, kernel=(1,1), pad=(0,0), stride=(1, 1), act_type='relu', separable=False) -# conv3 = conv3 + conv_dimred -# return conv3 - -def retina_detection_module(body, num_filter, filter_in, name): - assert num_filter%4==0 - conv1 = conv_act_layer(body, name+'_conv1', - num_filter//2, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='relu', separable=False, filter_in=filter_in) - conv2 = conv_act_layer(conv1, name+'_conv2', - num_filter//2, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='relu', separable=False, filter_in=num_filter//2) - conv3 = conv_act_layer(conv2, name+'_conv3', - num_filter//2, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='relu', separable=False, filter_in=num_filter//2) - conv4 = conv2 + conv3 - body = mx.sym.concat(*[conv1, conv4], dim=1, name = name+'_concat') - if config.USE_DCN>0: - body = conv_deformable(body, num_filter = num_filter, name = name+'_concat_dcn') - return body - - -def head_module(body, num_filter, filter_in, name): - if config.HEAD_MODULE=='SSH': - return ssh_detection_module(body, num_filter, filter_in, name) - else: - return retina_detection_module(body, num_filter, filter_in, name) - - -def upsampling(data, num_filter, name): - #ret = mx.symbol.Deconvolution(data=data, num_filter=num_filter, kernel=(4,4), stride=(2, 2), pad=(1,1), - # num_group = num_filter, no_bias = True, attr={'__lr_mult__': '0.0', '__wd_mult__': '0.0'}, - # name=name) - #ret = mx.symbol.Deconvolution(data=data, num_filter=num_filter, kernel=(2,2), stride=(2, 2), pad=(0,0), - # num_group = num_filter, no_bias = True, attr={'__lr_mult__': '0.0', '__wd_mult__': '0.0'}, - # name=name) - ret = mx.symbol.UpSampling(data, scale=2, sample_type='nearest', workspace=512, name=name, num_args=1) - return ret - -def get_sym_by_name(name, sym_buffer): - if name in sym_buffer: - return sym_buffer[name] - ret = None - name_key = name[0:1] - name_num = int(name[1:]) - #print('getting', name, name_key, name_num) - if name_key=='C': - assert name_num%2==0 - bottom = get_sym_by_name('C%d'%(name_num//2), sym_buffer) - ret = conv_act_layer(bottom, '%s_C%d'(PREFIX, name_num), - F1, kernel=(3, 3), pad=(1, 1), stride=(2, 2), act_type='relu', bias_wd_mult=_bwm) - elif name_key=='P': - assert name_num%2==0 - assert name_num<=max(config.RPN_FEAT_STRIDE) - lateral = get_sym_by_name('L%d'%(name_num), sym_buffer) - if name_num==max(config.RPN_FEAT_STRIDE) or name_num>32: - ret = mx.sym.identity(lateral, name='%s_P%d'%(PREFIX, name_num)) - else: - bottom = get_sym_by_name('L%d'%(name_num*2), sym_buffer) - bottom_up = upsampling(bottom, F1, '%s_U%d'%(PREFIX, name_num)) - if config.USE_CROP: - bottom_up = mx.symbol.Crop(*[bottom_up, lateral]) - aggr = lateral + bottom_up - aggr = conv_act_layer(aggr, '%s_A%d'%(PREFIX, name_num), - F1, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - ret = mx.sym.identity(aggr, name='%s_P%d'%(PREFIX, name_num)) - elif name_key=='L': - c = get_sym_by_name('C%d'%(name_num), sym_buffer) - #print('L', name, F1) - ret = conv_act_layer(c, '%s_L%d'%(PREFIX, name_num), - F1, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - else: - raise RuntimeError('%s is not a valid sym key name'%name) - sym_buffer[name] = ret - return ret - - -def get_sym_conv(data, sym): - all_layers = sym.get_internals() - - isize = 640 - _, out_shape, _ = all_layers.infer_shape(data = (1,3,isize,isize)) - last_entry = None - c1 = None - c2 = None - c3 = None - c1_name = None - c2_name = None - c3_name = None - c1_filter = -1 - c2_filter = -1 - c3_filter = -1 - #print(len(all_layers), len(out_shape)) - #print(all_layers.__class__) - outputs = all_layers.list_outputs() - #print(outputs.__class__, len(outputs)) - count = len(outputs) - stride2name = {} - stride2layer = {} - stride2shape = {} - for i in range(count): - name = outputs[i] - shape = out_shape[i] - print(i, name, count, shape) - if not name.endswith('_output'): - continue - if len(shape)!=4: - continue - assert isize%shape[2]==0 - if shape[1]>config.max_feat_channel: - break - stride = isize//shape[2] - stride2name[stride] = name - stride2layer[stride] = all_layers[name] - stride2shape[stride] = shape - - strides = sorted(stride2name.keys()) - for stride in strides: - print('stride', stride, stride2name[stride], stride2shape[stride]) - print('F1_F2', F1, F2) - #print('cnames', c1_name, c2_name, c3_name, F1, F2) - _bwm = 1.0 - ret = {} - sym_buffer = {} - for stride in [4,8,16,32]: - sym_buffer['C%d'%stride] = stride2layer[stride] - if not config.USE_FPN: - for stride in config.RPN_FEAT_STRIDE: - name = 'L%d'%stride - ret[stride] = get_sym_by_name(name, sym_buffer) - else: - for stride in config.RPN_FEAT_STRIDE: - name = 'P%d'%stride - ret[stride] = get_sym_by_name(name, sym_buffer) - - return ret - -def get_out(conv_fpn_feat, prefix, stride, landmark=False, lr_mult=1.0, gt_boxes=None): - A = config.NUM_ANCHORS - bbox_pred_len = 4 - landmark_pred_len = 10 - if config.USE_BLUR: - bbox_pred_len = 5 - if config.USE_OCCLUSION: - landmark_pred_len = 15 - ret_group = [] - num_anchors = config.RPN_ANCHOR_CFG[str(stride)]['NUM_ANCHORS'] - cls_label = mx.symbol.Variable(name='%s_label_stride%d'%(prefix,stride)) - bbox_target = mx.symbol.Variable(name='%s_bbox_target_stride%d'%(prefix,stride)) - bbox_weight = mx.symbol.Variable(name='%s_bbox_weight_stride%d'%(prefix,stride)) - if landmark: - landmark_target = mx.symbol.Variable(name='%s_landmark_target_stride%d'%(prefix,stride)) - landmark_weight = mx.symbol.Variable(name='%s_landmark_weight_stride%d'%(prefix,stride)) - conv_feat = conv_fpn_feat[stride] - rpn_relu = head_module(conv_feat, F2*config.CONTEXT_FILTER_RATIO, F1, 'rf_head_stride%d'%stride) - - rpn_cls_score = conv_only(rpn_relu, '%s_rpn_cls_score_stride%d'%(prefix, stride), 2*num_anchors, - kernel=(1,1), pad=(0,0), stride=(1, 1)) - - rpn_bbox_pred = conv_only(rpn_relu, '%s_rpn_bbox_pred_stride%d'%(prefix,stride), bbox_pred_len*num_anchors, - kernel=(1,1), pad=(0,0), stride=(1, 1)) - - # prepare rpn data - rpn_cls_score_reshape = mx.symbol.Reshape(data=rpn_cls_score, - shape=(0, 2, -1), - name="%s_rpn_cls_score_reshape_stride%s" % (prefix,stride)) - - rpn_bbox_pred_reshape = mx.symbol.Reshape(data=rpn_bbox_pred, - shape=(0, 0, -1), - name="%s_rpn_bbox_pred_reshape_stride%s" % (prefix,stride)) - if landmark: - rpn_landmark_pred = conv_only(rpn_relu, '%s_rpn_landmark_pred_stride%d'%(prefix,stride), landmark_pred_len*num_anchors, - kernel=(1,1), pad=(0,0), stride=(1, 1)) - rpn_landmark_pred_reshape = mx.symbol.Reshape(data=rpn_landmark_pred, - shape=(0, 0, -1), - name="%s_rpn_landmark_pred_reshape_stride%s" % (prefix,stride)) - - if config.TRAIN.RPN_ENABLE_OHEM>=2: - label, anchor_weight, pos_count = mx.sym.Custom(op_type='rpn_fpn_ohem3', stride=int(stride), network=config.network, dataset=config.dataset, prefix=prefix, cls_score=rpn_cls_score_reshape, labels = cls_label) - - _bbox_weight = mx.sym.tile(anchor_weight, (1,1,bbox_pred_len)) - _bbox_weight = _bbox_weight.reshape((0, -1, A * bbox_pred_len)).transpose((0,2,1)) - bbox_weight = mx.sym.elemwise_mul(bbox_weight, _bbox_weight, name='%s_bbox_weight_mul_stride%s'%(prefix,stride)) - - if landmark: - _landmark_weight = mx.sym.tile(anchor_weight, (1,1,landmark_pred_len)) - _landmark_weight = _landmark_weight.reshape((0, -1, A * landmark_pred_len)).transpose((0,2,1)) - landmark_weight = mx.sym.elemwise_mul(landmark_weight, _landmark_weight, name='%s_landmark_weight_mul_stride%s'%(prefix,stride)) - else: - label = cls_label - #if not config.FACE_LANDMARK: - # label, bbox_weight = mx.sym.Custom(op_type='rpn_fpn_ohem', stride=int(stride), cls_score=rpn_cls_score_reshape, bbox_weight = bbox_weight , labels = label) - #else: - # label, bbox_weight, landmark_weight = mx.sym.Custom(op_type='rpn_fpn_ohem2', stride=int(stride), cls_score=rpn_cls_score_reshape, bbox_weight = bbox_weight, landmark_weight=landmark_weight, labels = label) - #cls loss - rpn_cls_prob = mx.symbol.SoftmaxOutput(data=rpn_cls_score_reshape, - label=label, - multi_output=True, - normalization='valid', use_ignore=True, ignore_label=-1, - grad_scale = lr_mult, - name='%s_rpn_cls_prob_stride%d'%(prefix,stride)) - ret_group.append(rpn_cls_prob) - ret_group.append(mx.sym.BlockGrad(label)) - - pos_count = mx.symbol.sum(pos_count) - pos_count = pos_count + 0.001 #avoid zero - - #bbox loss - bbox_diff = rpn_bbox_pred_reshape-bbox_target - bbox_diff = bbox_diff * bbox_weight - rpn_bbox_loss_ = mx.symbol.smooth_l1(name='%s_rpn_bbox_loss_stride%d_'%(prefix,stride), scalar=3.0, data=bbox_diff) - bbox_lr_mode0 = 0.25*lr_mult*config.TRAIN.BATCH_IMAGES / config.TRAIN.RPN_BATCH_SIZE - landmark_lr_mode0 = 0.4*config.LANDMARK_LR_MULT*bbox_lr_mode0 - if config.LR_MODE==0: - rpn_bbox_loss = mx.sym.MakeLoss(name='%s_rpn_bbox_loss_stride%d'%(prefix,stride), data=rpn_bbox_loss_, grad_scale=bbox_lr_mode0) - else: - rpn_bbox_loss_ = mx.symbol.broadcast_div(rpn_bbox_loss_, pos_count) - rpn_bbox_loss = mx.sym.MakeLoss(name='%s_rpn_bbox_loss_stride%d'%(prefix,stride), data=rpn_bbox_loss_, grad_scale=0.5*lr_mult) - ret_group.append(rpn_bbox_loss) - ret_group.append(mx.sym.BlockGrad(bbox_weight)) - - #landmark loss - if landmark: - landmark_diff = rpn_landmark_pred_reshape-landmark_target - landmark_diff = landmark_diff * landmark_weight - rpn_landmark_loss_ = mx.symbol.smooth_l1(name='%s_rpn_landmark_loss_stride%d_'%(prefix,stride), scalar=3.0, data=landmark_diff) - if config.LR_MODE==0: - rpn_landmark_loss = mx.sym.MakeLoss(name='%s_rpn_landmark_loss_stride%d'%(prefix,stride), data=rpn_landmark_loss_, grad_scale=landmark_lr_mode0) - else: - rpn_landmark_loss_ = mx.symbol.broadcast_div(rpn_landmark_loss_, pos_count) - rpn_landmark_loss = mx.sym.MakeLoss(name='%s_rpn_landmark_loss_stride%d'%(prefix,stride), data=rpn_landmark_loss_, grad_scale=0.2*config.LANDMARK_LR_MULT*lr_mult) - ret_group.append(rpn_landmark_loss) - ret_group.append(mx.sym.BlockGrad(landmark_weight)) - if config.USE_3D: - from rcnn.PY_OP import rpn_3d_mesh - pass - if config.CASCADE>0: - if config.CASCADE_MODE==0: - body = rpn_relu - elif config.CASCADE_MODE==1: - body = head_module(conv_feat, F2*config.CONTEXT_FILTER_RATIO, F1, '%s_head_stride%d_cas'%(PREFIX, stride)) - elif config.CASCADE_MODE==2: - body = conv_feat + rpn_relu - body = head_module(body, F2*config.CONTEXT_FILTER_RATIO, F1, '%s_head_stride%d_cas'%(PREFIX,stride)) - else: - body = head_module(conv_feat, F2*config.CONTEXT_FILTER_RATIO, F1, '%s_head_stride%d_cas'%(PREFIX, stride)) - body = mx.sym.concat(body, rpn_cls_score, rpn_bbox_pred, rpn_landmark_pred, dim=1) - - #cls_pred = rpn_cls_prob - cls_pred_t0 = rpn_cls_score_reshape - cls_label_raw = cls_label - cls_label_t0 = label - bbox_pred_t0 = rpn_bbox_pred_reshape - #bbox_pred = rpn_bbox_pred - #bbox_pred = mx.sym.transpose(bbox_pred, (0, 2, 3, 1)) - #bbox_pred_len = 4 - #bbox_pred = mx.sym.reshape(bbox_pred, (0, -1, bbox_pred_len)) - bbox_label_t0 = bbox_target - #prefix = prefix+'2' - for casid in range(config.CASCADE): - #pseudo-code - #anchor_label = GENANCHOR(bbox_label, bbox_pred, stride) - #bbox_label = F(anchor_label, bbox_pred) - #bbox_label = bbox_label - bbox_pred - cls_pred = conv_only(body, '%s_rpn_cls_score_stride%d_cas%d'%(prefix, stride, casid), 2*num_anchors, - kernel=(1,1), pad=(0,0), stride=(1, 1)) - rpn_cls_score_reshape = mx.symbol.Reshape(data=cls_pred, - shape=(0, 2, -1), - name="%s_rpn_cls_score_reshape_stride%s_cas%d" % (prefix,stride, casid)) - - #bbox_label equals to bbox_target - #cls_pred, cls_label, bbox_pred, bbox_label, bbox_weight, pos_count = mx.sym.Custom(op_type='cascade_refine', stride=int(stride), network=config.network, dataset=config.dataset, prefix=prefix, cls_pred=cls_pred, cls_label = cls_label, bbox_pred = bbox_pred, bbox_label = bbox_label) - #cls_label, bbox_label, anchor_weight, pos_count = mx.sym.Custom(op_type='cascade_refine', stride=int(stride), network=config.network, dataset=config.dataset, prefix=prefix, cls_pred_t0=cls_pred_t0, cls_label_t0 = cls_label_t0, cls_pred = rpn_cls_score_reshape, bbox_pred_t0 = bbox_pred_t0, bbox_label_t0 = bbox_label_t0) - cls_label, bbox_label, anchor_weight, pos_count = mx.sym.Custom(op_type='cascade_refine', stride=int(stride), network=config.network, - dataset=config.dataset, prefix=prefix, - cls_label_t0 = cls_label_t0, cls_pred_t0=cls_pred_t0, cls_pred = rpn_cls_score_reshape, - bbox_pred_t0 = bbox_pred_t0, bbox_label_t0 = bbox_label_t0, - cls_label_raw = cls_label_raw, cas_gt_boxes = gt_boxes) - if stride in config.CASCADE_CLS_STRIDES: - rpn_cls_prob = mx.symbol.SoftmaxOutput(data=rpn_cls_score_reshape, - label=cls_label, - multi_output=True, - normalization='valid', use_ignore=True, ignore_label=-1, - grad_scale = lr_mult, - name='%s_rpn_cls_prob_stride%d_cas%d'%(prefix,stride,casid)) - ret_group.append(rpn_cls_prob) - ret_group.append(mx.sym.BlockGrad(cls_label)) - if stride in config.CASCADE_BBOX_STRIDES: - bbox_pred = conv_only(body, '%s_rpn_bbox_pred_stride%d_cas%d'%(prefix,stride,casid), bbox_pred_len*num_anchors, - kernel=(1,1), pad=(0,0), stride=(1, 1)) - - rpn_bbox_pred_reshape = mx.symbol.Reshape(data=bbox_pred, - shape=(0, 0, -1), - name="%s_rpn_bbox_pred_reshape_stride%s_cas%d" % (prefix,stride,casid)) - _bbox_weight = mx.sym.tile(anchor_weight, (1,1,bbox_pred_len)) - _bbox_weight = _bbox_weight.reshape((0, -1, A * bbox_pred_len)).transpose((0,2,1)) - bbox_weight = _bbox_weight - pos_count = mx.symbol.sum(pos_count) - pos_count = pos_count + 0.01 #avoid zero - #bbox_weight = mx.sym.elemwise_mul(bbox_weight, _bbox_weight, name='%s_bbox_weight_mul_stride%s'%(prefix,stride)) - #bbox loss - bbox_diff = rpn_bbox_pred_reshape-bbox_label - bbox_diff = bbox_diff * bbox_weight - rpn_bbox_loss_ = mx.symbol.smooth_l1(name='%s_rpn_bbox_loss_stride%d_cas%d'%(prefix,stride,casid), scalar=3.0, data=bbox_diff) - if config.LR_MODE==0: - rpn_bbox_loss = mx.sym.MakeLoss(name='%s_rpn_bbox_loss_stride%d_cas%d'%(prefix,stride,casid), data=rpn_bbox_loss_, grad_scale=bbox_lr_mode0) - else: - rpn_bbox_loss_ = mx.symbol.broadcast_div(rpn_bbox_loss_, pos_count) - rpn_bbox_loss = mx.sym.MakeLoss(name='%s_rpn_bbox_loss_stride%d_cas%d'%(prefix,stride,casid), data=rpn_bbox_loss_, grad_scale=0.5*lr_mult) - ret_group.append(rpn_bbox_loss) - ret_group.append(mx.sym.BlockGrad(bbox_weight)) - #bbox_pred = rpn_bbox_pred_reshape - - return ret_group - -def get_sym_train(sym): - data = mx.symbol.Variable(name="data") - global F1, F2 - F1 = config.HEAD_FILTER_NUM - F2 = F1 - - # shared convolutional layers - conv_fpn_feat = get_sym_conv(data, sym) - ret_group = [] - gt_boxes = None - if config.CASCADE>0: - gt_boxes = mx.sym.Variable('gt_boxes') - - - for stride in config.RPN_FEAT_STRIDE: - ret = get_out(conv_fpn_feat, 'face', stride, config.FACE_LANDMARK, lr_mult=1.0, gt_boxes = gt_boxes) - ret_group += ret - - return mx.sym.Group(ret_group) - - diff --git a/RetinaFace/rcnn/symbol/symbol_mnet.py b/RetinaFace/rcnn/symbol/symbol_mnet.py deleted file mode 100644 index 4a8e2b5..0000000 --- a/RetinaFace/rcnn/symbol/symbol_mnet.py +++ /dev/null @@ -1,492 +0,0 @@ -import mxnet as mx -import mxnet.ndarray as nd -import mxnet.gluon as gluon -import mxnet.gluon.nn as nn -import mxnet.autograd as ag -import numpy as np -from rcnn.config import config -from rcnn.PY_OP import rpn_fpn_ohem3 -from rcnn.symbol.symbol_common import get_sym_train - - -def conv_only(from_layer, name, num_filter, kernel=(1,1), pad=(0,0), \ - stride=(1,1), bias_wd_mult=0.0, shared_weight=None, shared_bias = None): - if shared_weight is None: - weight = mx.symbol.Variable(name="{}_weight".format(name), - init=mx.init.Normal(0.01), attr={'__lr_mult__': '1.0'}) - bias = mx.symbol.Variable(name="{}_bias".format(name), - init=mx.init.Constant(0.0), attr={'__lr_mult__': '2.0', '__wd_mult__': str(bias_wd_mult)}) - else: - weight = shared_weight - bias = shared_bias - print('reuse shared var in', name) - conv = mx.symbol.Convolution(data=from_layer, kernel=kernel, pad=pad, \ - stride=stride, num_filter=num_filter, name="{}".format(name), weight = weight, bias=bias) - return conv - -def conv_act_layer_dw(from_layer, name, num_filter, kernel=(1,1), pad=(0,0), \ - stride=(1,1), act_type="relu", bias_wd_mult=0.0): - assert kernel[0]==3 - weight = mx.symbol.Variable(name="{}_weight".format(name), - init=mx.init.Normal(0.01), attr={'__lr_mult__': '1.0'}) - bias = mx.symbol.Variable(name="{}_bias".format(name), - init=mx.init.Constant(0.0), attr={'__lr_mult__': '2.0', '__wd_mult__': str(bias_wd_mult)}) - conv = mx.symbol.Convolution(data=from_layer, kernel=kernel, pad=pad, \ - stride=stride, num_filter=num_filter, num_group=num_filter, name="{}".format(name), weight=weight, bias=bias) - conv = mx.sym.BatchNorm(data=conv, fix_gamma=False, eps=2e-5, momentum=0.9, name=name + '_bn') - if len(act_type)>0: - relu = mx.symbol.Activation(data=conv, act_type=act_type, \ - name="{}_{}".format(name, act_type)) - else: - relu = conv - return relu - -def conv_act_layer(from_layer, name, num_filter, kernel=(1,1), pad=(0,0), \ - stride=(1,1), act_type="relu", bias_wd_mult=0.0, separable=False, filter_in = -1): - - separable = False - if separable: - assert kernel[0]==3 - if not separable: - weight = mx.symbol.Variable(name="{}_weight".format(name), - init=mx.init.Normal(0.01), attr={'__lr_mult__': '1.0'}) - bias = mx.symbol.Variable(name="{}_bias".format(name), - init=mx.init.Constant(0.0), attr={'__lr_mult__': '2.0', '__wd_mult__': str(bias_wd_mult)}) - conv = mx.symbol.Convolution(data=from_layer, kernel=kernel, pad=pad, \ - stride=stride, num_filter=num_filter, name="{}".format(name), weight=weight, bias=bias) - conv = mx.sym.BatchNorm(data=conv, fix_gamma=False, eps=2e-5, momentum=0.9, name=name + '_bn') - else: - if filter_in<0: - filter_in = num_filter - conv = mx.symbol.Convolution(data=from_layer, kernel=kernel, pad=pad, \ - stride=stride, num_filter=filter_in, num_group=filter_in, name="{}_sep".format(name)) - conv = mx.sym.BatchNorm(data=conv, fix_gamma=False, eps=2e-5, momentum=0.9, name=name + '_sep_bn') - conv = mx.symbol.Activation(data=conv, act_type='relu', \ - name="{}_sep_bn_relu".format(name)) - conv = mx.symbol.Convolution(data=conv, kernel=(1,1), pad=(0,0), \ - stride=(1,1), num_filter=num_filter, name="{}".format(name)) - conv = mx.sym.BatchNorm(data=conv, fix_gamma=False, eps=2e-5, momentum=0.9, name=name + '_bn') - if len(act_type)>0: - relu = mx.symbol.Activation(data=conv, act_type=act_type, \ - name="{}_{}".format(name, act_type)) - else: - relu = conv - return relu - -def ssh_context_module(body, num_filter, filter_in, name): - conv_dimred = conv_act_layer(body, name+'_conv1', - num_filter, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='relu', separable=True, filter_in = filter_in) - conv5x5 = conv_act_layer(conv_dimred, name+'_conv2', - num_filter, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='', separable=True) - conv7x7_1 = conv_act_layer(conv_dimred, name+'_conv3_1', - num_filter, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='relu', separable=True) - conv7x7 = conv_act_layer(conv7x7_1, name+'_conv3_2', - num_filter, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='', separable=True) - return (conv5x5, conv7x7) - -def ssh_detection_module(body, num_filter, filter_in, name): - conv3x3 = conv_act_layer(body, name+'_conv1', - num_filter, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='', separable=True, filter_in=filter_in) - conv5x5, conv7x7 = ssh_context_module(body, num_filter//2, filter_in, name+'_context') - ret = mx.sym.concat(*[conv3x3, conv5x5, conv7x7], dim=1, name = name+'_concat') - ret = mx.symbol.Activation(data=ret, act_type='relu', name=name+'_concat_relu') - return ret - - -def upsampling(data, num_filter, name): - #ret = mx.symbol.Deconvolution(data=data, num_filter=num_filter, kernel=(4,4), stride=(2, 2), pad=(1,1), - # num_group = num_filter, no_bias = True, attr={'__lr_mult__': '0.0', '__wd_mult__': '0.0'}, - # name=name) - #ret = mx.symbol.Deconvolution(data=data, num_filter=num_filter, kernel=(2,2), stride=(2, 2), pad=(0,0), - # num_group = num_filter, no_bias = True, attr={'__lr_mult__': '0.0', '__wd_mult__': '0.0'}, - # name=name) - ret = mx.symbol.UpSampling(data, scale=2, sample_type='nearest', workspace=512, name=name, num_args=1) - return ret - -def get_mnet_conv(data, sym): - mm = config.MULTIPLIER - all_layers = sym.get_internals() - #print(all_layers) - ##c1 = all_layers['mobilenetv20_features_linearbottleneck6_relu60_relu6_output'] #96 - #c1 = all_layers['mobilenetv20_features_linearbottleneck5_elemwise_add0_output'] # 16 - ##c2 = all_layers['mobilenetv20_features_linearbottleneck13_relu60_relu6_output'] - #c2 = all_layers['mobilenetv20_features_linearbottleneck12_elemwise_add0_output'] # 48 - ##c3 = all_layers['mobilenetv20_features_linearbottleneck16_batchnorm2_fwd_output'] # 160 - #c3 = all_layers['mobilenetv20_features_linearbottleneck13_batchnorm2_fwd_output'] # 80 - #c1_filter = int(32*mm) - #c2_filter = int(96*mm) - #c3_filter = int(160*mm) - - #c1 = all_layers['mobilenet0_relu10_fwd_output'] - #c2 = all_layers['mobilenet0_relu22_fwd_output'] - #c3 = all_layers['mobilenet0_relu26_fwd_output'] - - #c1 = all_layers['conv_6_relu_output'] - #c2 = all_layers['conv_12_relu_output'] - #c3 = all_layers['conv_14_relu_output'] - #c1_filter = int(256*mm) - #c2_filter = int(512*mm) - #c3_filter = int(1024*mm) - - isize = 640 - _, out_shape, _ = all_layers.infer_shape(data = (1,3,isize,isize)) - last_entry = None - c1 = None - c2 = None - c3 = None - c1_name = None - c2_name = None - c3_name = None - c1_filter = -1 - c2_filter = -1 - c3_filter = -1 - #print(len(all_layers), len(out_shape)) - #print(all_layers.__class__) - outputs = all_layers.list_outputs() - #print(outputs.__class__, len(outputs)) - count = len(outputs) - for i in range(count): - name = outputs[i] - shape = out_shape[i] - if not name.endswith('_output'): - continue - if len(shape)!=4: - continue - #print(name, shape) - if c1 is None and shape[2]==isize//16: - cname = last_entry[0] - #print('c1', last_entry) - c1 = all_layers[cname] - c1_name = cname - if c2 is None and shape[2]==isize//32: - cname = last_entry[0] - #print('c2', last_entry) - c2 = all_layers[cname] - c2_name = cname - if shape[2]==isize//32: - c3 = all_layers[name] - #print('c3', name, shape) - c3_name = name - - last_entry = (name, shape) - print('cnames', c1_name, c2_name, c3_name) - - F1 = int(256*mm) - F2 = int(128*mm) - if config.SHARE_WEIGHT_BBOX or config.SHARE_WEIGHT_LANDMARK: - F2 = F1 - _bwm = 1.0 - if config.NET_MODE==0: - c1_lateral = conv_act_layer(c1, 'ssh_m1_red_conv', - F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - c2_lateral = conv_act_layer(c2, 'ssh_m2_red_conv', - F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - #conv5_128_up = mx.symbol.Deconvolution(data=conv5_128, num_filter=F2, kernel=(4,4), stride=(2, 2), pad=(1,1), - # num_group = F2, no_bias = True, attr={'__lr_mult__': '0.0', '__wd_mult__': '0.0'}, - # name='ssh_m2_red_upsampling') - #c2_up = mx.symbol.UpSampling(c2_lateral, scale=2, sample_type='nearest', workspace=512, name='ssh_m2_red_up', num_args=1) - c2_up = upsampling(c2_lateral, F2, 'ssh_m2_red_upsampling') - #conv4_128 = mx.symbol.Crop(*[conv4_128, conv5_128_up]) - c2_up = mx.symbol.Crop(*[c2_up, c1_lateral]) - - c1 = c1_lateral+c2_up - - c1 = conv_act_layer(c1, 'ssh_m1_conv', - F2, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - m1 = ssh_detection_module(c1, F2, F2, 'ssh_m1_det') - m2 = ssh_detection_module(c2, F1, c2_filter, 'ssh_m2_det') - m3 = ssh_detection_module(c3, F1, c3_filter, 'ssh_m3_det') - elif config.NET_MODE==1: - c3_lateral = conv_act_layer(c3, 'ssh_c3_lateral', - F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - #c3_up = mx.symbol.UpSampling(c3_lateral, scale=2, sample_type='nearest', workspace=512, name='ssh_c3_up', num_args=1) - c3_up = upsampling(c3_lateral, F2, 'ssh_c3_upsampling') - c2_lateral = conv_act_layer(c2, 'ssh_c2_lateral', - F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - c3_up = mx.symbol.Crop(*[c3_up, c2_lateral]) - c2 = c2_lateral+c3_up - c2 = conv_act_layer(c2, 'ssh_c2_aggr', - F2, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - c1_lateral = conv_act_layer(c1, 'ssh_m1_red_conv', - F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - #c2_up = mx.symbol.UpSampling(c2, scale=2, sample_type='nearest', workspace=512, name='ssh_m2_red_up', num_args=1) - c2_up = upsampling(c2, F2, 'ssh_c2_upsampling') - #conv4_128 = mx.symbol.Crop(*[conv4_128, conv5_128_up]) - c2_up = mx.symbol.Crop(*[c2_up, c1_lateral]) - - c1 = c1_lateral+c2_up - - c1 = conv_act_layer(c1, 'ssh_m1_conv', - F2, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - m1 = ssh_detection_module(c1, F2, F2, 'ssh_m1_det') - m2 = ssh_detection_module(c2, F1, c2_filter, 'ssh_m2_det') - m3 = ssh_detection_module(c3, F1, c3_filter, 'ssh_m3_det') - elif config.NET_MODE==2: - c3 = conv_act_layer(c3, 'ssh_c3_lateral', - F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - #c3_up = mx.symbol.UpSampling(c3, scale=2, sample_type='nearest', workspace=512, name='ssh_c3_up', num_args=1) - c3_up = upsampling(c3, F2, 'ssh_c3_upsampling') - c2_lateral = conv_act_layer(c2, 'ssh_c2_lateral', - F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - c3_up = mx.symbol.Crop(*[c3_up, c2_lateral]) - c2 = c2_lateral+c3_up - c2 = conv_act_layer(c2, 'ssh_c2_aggr', - F2, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - c1_lateral = conv_act_layer(c1, 'ssh_m1_red_conv', - F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - #c2_up = mx.symbol.UpSampling(c2, scale=2, sample_type='nearest', workspace=512, name='ssh_m2_red_up', num_args=1) - c2_up = upsampling(c2, F2, 'ssh_c2_upsampling') - #conv4_128 = mx.symbol.Crop(*[conv4_128, conv5_128_up]) - c2_up = mx.symbol.Crop(*[c2_up, c1_lateral]) - c1 = c1_lateral+c2_up - c1 = conv_act_layer(c1, 'ssh_c1_aggr', - F2, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - - m1 = ssh_detection_module(c1, F2, F2, 'ssh_m1_det') - m2 = ssh_detection_module(c2, F1, c2_filter, 'ssh_m2_det') - m3 = ssh_detection_module(c3, F1, c3_filter, 'ssh_m3_det') - elif config.NET_MODE==3: - #c3 = conv_act_layer(c3, 'ssh_c3_lateral', - # F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - c3 = ssh_detection_module(c3, F2//2, c3_filter, 'ssh_c3_lateral') - #c3_up = mx.symbol.UpSampling(c3, scale=2, sample_type='nearest', workspace=512, name='ssh_c3_up', num_args=1) - c3_up = upsampling(c3, F2, 'ssh_c3_upsampling') - #c2_lateral = conv_act_layer(c2, 'ssh_c2_lateral', - # F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - c2_lateral = ssh_detection_module(c2, F2//2, c2_filter, 'ssh_c2_lateral') - c3_up = mx.symbol.Crop(*[c3_up, c2_lateral]) - c2 = c2_lateral+c3_up - c2 = conv_act_layer(c2, 'ssh_c2_aggr', - F2, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - #c1_lateral = conv_act_layer(c1, 'ssh_m1_red_conv', - # F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - c1_lateral = ssh_detection_module(c1, F2//2, c1_filter, 'ssh_c1_lateral') - #c2_up = mx.symbol.UpSampling(c2, scale=2, sample_type='nearest', workspace=512, name='ssh_m2_red_up', num_args=1) - c2_up = upsampling(c2, F2, 'ssh_c2_upsampling') - #conv4_128 = mx.symbol.Crop(*[conv4_128, conv5_128_up]) - c2_up = mx.symbol.Crop(*[c2_up, c1_lateral]) - c1 = c1_lateral+c2_up - c1 = conv_act_layer(c1, 'ssh_c1_aggr', - F2, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - - m1 = ssh_detection_module(c1, F2, F2, 'ssh_m1_det') - m2 = ssh_detection_module(c2, F1, c2_filter, 'ssh_m2_det') - m3 = ssh_detection_module(c3, F1, c3_filter, 'ssh_m3_det') - elif config.NET_MODE==4: - c3 = conv_act_layer(c3, 'ssh_c3_lateral', - F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - #c3_up = mx.symbol.UpSampling(c3, scale=2, sample_type='nearest', workspace=512, name='ssh_c3_up', num_args=1) - c3_up = upsampling(c3, F2, 'ssh_c3_upsampling') - c2_lateral = conv_act_layer(c2, 'ssh_c2_lateral', - F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - c3_up = mx.symbol.Crop(*[c3_up, c2_lateral]) - c2 = c2_lateral+c3_up - c2 = conv_act_layer(c2, 'ssh_c2_aggr', - F2, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - c1_lateral = conv_act_layer(c1, 'ssh_m1_red_conv', - F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - #c2_up = mx.symbol.UpSampling(c2, scale=2, sample_type='nearest', workspace=512, name='ssh_m2_red_up', num_args=1) - c2_up = upsampling(c2, F2, 'ssh_c2_upsampling') - #conv4_128 = mx.symbol.Crop(*[conv4_128, conv5_128_up]) - c2_up = mx.symbol.Crop(*[c2_up, c1_lateral]) - c1 = c1_lateral+c2_up - c1 = conv_act_layer(c1, 'ssh_c1_aggr', - F2, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - - m1 = ssh_detection_module(c1, F2//2, F2, 'ssh_m1_det') - m2 = ssh_detection_module(c2, F1//2, c2_filter, 'ssh_m2_det') - m3 = ssh_detection_module(c3, F1//2, c3_filter, 'ssh_m3_det') - elif config.NET_MODE==5: - c3 = conv_act_layer_dw(c3, 'ssh_c3_lateral_m', - F2, kernel=(3,3), pad=(1,1), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - c3 = conv_act_layer(c3, 'ssh_c3_lateral', - F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - #c3_up = mx.symbol.UpSampling(c3, scale=2, sample_type='nearest', workspace=512, name='ssh_c3_up', num_args=1) - c3_up = upsampling(c3, F2, 'ssh_c3_upsampling') - c2 = conv_act_layer_dw(c2, 'ssh_c2_lateral_m', - F2, kernel=(3,3), pad=(1,1), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - c2_lateral = conv_act_layer(c2, 'ssh_c2_lateral', - F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - c3_up = mx.symbol.Crop(*[c3_up, c2_lateral]) - c2 = c2_lateral+c3_up - c2 = conv_act_layer(c2, 'ssh_c2_aggr', - F2, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - c1 = conv_act_layer_dw(c1, 'ssh_c1_lateral_m', - F2, kernel=(3,3), pad=(1,1), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - c1_lateral = conv_act_layer(c1, 'ssh_m1_red_conv', - F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - #c2_up = mx.symbol.UpSampling(c2, scale=2, sample_type='nearest', workspace=512, name='ssh_m2_red_up', num_args=1) - c2_up = upsampling(c2, F2, 'ssh_c2_upsampling') - #conv4_128 = mx.symbol.Crop(*[conv4_128, conv5_128_up]) - c2_up = mx.symbol.Crop(*[c2_up, c1_lateral]) - c1 = c1_lateral+c2_up - c1 = conv_act_layer(c1, 'ssh_c1_aggr', - F2, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - - m1 = ssh_detection_module(c1, F2, F2, 'ssh_m1_det') - m2 = ssh_detection_module(c2, F1, c2_filter, 'ssh_m2_det') - m3 = ssh_detection_module(c3, F1, c3_filter, 'ssh_m3_det') - - return {8: m1, 16:m2, 32: m3} - -def get_out(conv_fpn_feat, prefix, stride, landmark=False, lr_mult=1.0, shared_vars = None): - A = config.NUM_ANCHORS - bbox_pred_len = 4 - landmark_pred_len = 10 - if config.USE_BLUR: - bbox_pred_len = 5 - if config.USE_OCCLUSION: - landmark_pred_len = 15 - ret_group = [] - num_anchors = config.RPN_ANCHOR_CFG[str(stride)]['NUM_ANCHORS'] - label = mx.symbol.Variable(name='%s_label_stride%d'%(prefix,stride)) - bbox_target = mx.symbol.Variable(name='%s_bbox_target_stride%d'%(prefix,stride)) - bbox_weight = mx.symbol.Variable(name='%s_bbox_weight_stride%d'%(prefix,stride)) - if landmark: - landmark_target = mx.symbol.Variable(name='%s_landmark_target_stride%d'%(prefix,stride)) - landmark_weight = mx.symbol.Variable(name='%s_landmark_weight_stride%d'%(prefix,stride)) - rpn_relu = conv_fpn_feat[stride] - maxout_stat = 0 - if config.USE_MAXOUT>=1 and stride==config.RPN_FEAT_STRIDE[-1]: - maxout_stat = 1 - if config.USE_MAXOUT>=2 and stride!=config.RPN_FEAT_STRIDE[-1]: - maxout_stat = 2 - - - if maxout_stat==0: - rpn_cls_score = conv_only(rpn_relu, '%s_rpn_cls_score_stride%d'%(prefix, stride), 2*num_anchors, - kernel=(1,1), pad=(0,0), stride=(1, 1), shared_weight = shared_vars[0][0], shared_bias = shared_vars[0][1]) - elif maxout_stat==1: - cls_list = [] - for a in range(num_anchors): - rpn_cls_score_bg = conv_only(rpn_relu, '%s_rpn_cls_score_stride%d_anchor%d_bg'%(prefix,stride,a), 3, - kernel=(1,1), pad=(0,0), stride=(1, 1)) - rpn_cls_score_bg = mx.sym.max(rpn_cls_score_bg, axis=1, keepdims=True) - cls_list.append(rpn_cls_score_bg) - rpn_cls_score_fg = conv_only(rpn_relu, '%s_rpn_cls_score_stride%d_anchor%d_fg'%(prefix,stride,a), 1, - kernel=(1,1), pad=(0,0), stride=(1, 1)) - cls_list.append(rpn_cls_score_fg) - rpn_cls_score = mx.sym.concat(*cls_list, dim=1, name='%s_rpn_cls_score_stride%d'%(prefix,stride)) - else: - cls_list = [] - for a in range(num_anchors): - rpn_cls_score_bg = conv_only(rpn_relu, '%s_rpn_cls_score_stride%d_anchor%d_bg'%(prefix,stride,a), 1, - kernel=(1,1), pad=(0,0), stride=(1, 1)) - cls_list.append(rpn_cls_score_bg) - rpn_cls_score_fg = conv_only(rpn_relu, '%s_rpn_cls_score_stride%d_anchor%d_fg'%(prefix,stride,a), 3, - kernel=(1,1), pad=(0,0), stride=(1, 1)) - rpn_cls_score_fg = mx.sym.max(rpn_cls_score_fg, axis=1, keepdims=True) - cls_list.append(rpn_cls_score_fg) - rpn_cls_score = mx.sym.concat(*cls_list, dim=1, name='%s_rpn_cls_score_stride%d'%(prefix,stride)) - - rpn_bbox_pred = conv_only(rpn_relu, '%s_rpn_bbox_pred_stride%d'%(prefix,stride), bbox_pred_len*num_anchors, - kernel=(1,1), pad=(0,0), stride=(1, 1), shared_weight = shared_vars[1][0], shared_bias = shared_vars[1][1]) - - # prepare rpn data - if not config.FBN: - rpn_cls_score_reshape = mx.symbol.Reshape(data=rpn_cls_score, - shape=(0, 2, -1), - name="%s_rpn_cls_score_reshape_stride%s" % (prefix,stride)) - else: - rpn_cls_score_reshape = mx.symbol.Reshape(data=rpn_cls_score, - shape=(0, 2, -1), - name="%s_rpn_cls_score_reshape_stride%s_pre" % (prefix,stride)) - rpn_cls_score_reshape = mx.symbol.BatchNorm(rpn_cls_score_reshape, fix_gamma=True, eps=2e-5, name="%s_rpn_cls_score_reshape_stride%s"%(prefix, stride)) - - rpn_bbox_pred_reshape = mx.symbol.Reshape(data=rpn_bbox_pred, - shape=(0, 0, -1), - name="%s_rpn_bbox_pred_reshape_stride%s" % (prefix,stride)) - if landmark: - rpn_landmark_pred = conv_only(rpn_relu, '%s_rpn_landmark_pred_stride%d'%(prefix,stride), landmark_pred_len*num_anchors, - kernel=(1,1), pad=(0,0), stride=(1, 1), shared_weight = shared_vars[2][0], shared_bias = shared_vars[2][1]) - rpn_landmark_pred_reshape = mx.symbol.Reshape(data=rpn_landmark_pred, - shape=(0, 0, -1), - name="%s_rpn_landmark_pred_reshape_stride%s" % (prefix,stride)) - - if config.TRAIN.RPN_ENABLE_OHEM>=2: - label, anchor_weight = mx.sym.Custom(op_type='rpn_fpn_ohem3', stride=int(stride), network=config.network, dataset=config.dataset, prefix=prefix, cls_score=rpn_cls_score_reshape, labels = label) - - _bbox_weight = mx.sym.tile(anchor_weight, (1,1,bbox_pred_len)) - _bbox_weight = _bbox_weight.reshape((0, -1, A * bbox_pred_len)).transpose((0,2,1)) - bbox_weight = mx.sym.elemwise_mul(bbox_weight, _bbox_weight, name='%s_bbox_weight_mul_stride%s'%(prefix,stride)) - - if landmark: - _landmark_weight = mx.sym.tile(anchor_weight, (1,1,landmark_pred_len)) - _landmark_weight = _landmark_weight.reshape((0, -1, A * landmark_pred_len)).transpose((0,2,1)) - landmark_weight = mx.sym.elemwise_mul(landmark_weight, _landmark_weight, name='%s_landmark_weight_mul_stride%s'%(prefix,stride)) - #if not config.FACE_LANDMARK: - # label, bbox_weight = mx.sym.Custom(op_type='rpn_fpn_ohem', stride=int(stride), cls_score=rpn_cls_score_reshape, bbox_weight = bbox_weight , labels = label) - #else: - # label, bbox_weight, landmark_weight = mx.sym.Custom(op_type='rpn_fpn_ohem2', stride=int(stride), cls_score=rpn_cls_score_reshape, bbox_weight = bbox_weight, landmark_weight=landmark_weight, labels = label) - #cls loss - rpn_cls_prob = mx.symbol.SoftmaxOutput(data=rpn_cls_score_reshape, - label=label, - multi_output=True, - normalization='valid', use_ignore=True, ignore_label=-1, - grad_scale = lr_mult, - name='%s_rpn_cls_prob_stride%d'%(prefix,stride)) - ret_group.append(rpn_cls_prob) - ret_group.append(mx.sym.BlockGrad(label)) - - #bbox loss - bbox_diff = rpn_bbox_pred_reshape-bbox_target - bbox_diff = bbox_diff * bbox_weight - rpn_bbox_loss_ = mx.symbol.smooth_l1(name='%s_rpn_bbox_loss_stride%d_'%(prefix,stride), scalar=3.0, data=bbox_diff) - rpn_bbox_loss = mx.sym.MakeLoss(name='%s_rpn_bbox_loss_stride%d'%(prefix,stride), data=rpn_bbox_loss_, grad_scale=1.0*lr_mult / (config.TRAIN.RPN_BATCH_SIZE)) - ret_group.append(rpn_bbox_loss) - ret_group.append(mx.sym.BlockGrad(bbox_weight)) - - #landmark loss - if landmark: - landmark_diff = rpn_landmark_pred_reshape-landmark_target - landmark_diff = landmark_diff * landmark_weight - rpn_landmark_loss_ = mx.symbol.smooth_l1(name='%s_rpn_landmark_loss_stride%d_'%(prefix,stride), scalar=3.0, data=landmark_diff) - rpn_landmark_loss = mx.sym.MakeLoss(name='%s_rpn_landmark_loss_stride%d'%(prefix,stride), data=rpn_landmark_loss_, grad_scale=0.5*lr_mult / (config.TRAIN.RPN_BATCH_SIZE)) - ret_group.append(rpn_landmark_loss) - ret_group.append(mx.sym.BlockGrad(landmark_weight)) - return ret_group - -def get_mnet_train(sym): - return get_sym_train(sym) - #data = mx.symbol.Variable(name="data") - ## shared convolutional layers - #conv_fpn_feat = get_mnet_conv(data, sym) - #ret_group = [] - #shared_vars = [] - #if config.SHARE_WEIGHT_BBOX: - # assert config.USE_MAXOUT==0 - # _name = 'face_rpn_cls_score_share' - # shared_weight = mx.symbol.Variable(name="{}_weight".format(_name), - # init=mx.init.Normal(0.01), attr={'__lr_mult__': '1.0'}) - # shared_bias = mx.symbol.Variable(name="{}_bias".format(_name), - # init=mx.init.Constant(0.0), attr={'__lr_mult__': '2.0', '__wd_mult__': str(0.0)}) - # shared_vars.append( [shared_weight, shared_bias] ) - # _name = 'face_rpn_bbox_pred_share' - # shared_weight = mx.symbol.Variable(name="{}_weight".format(_name), - # init=mx.init.Normal(0.01), attr={'__lr_mult__': '1.0'}) - # shared_bias = mx.symbol.Variable(name="{}_bias".format(_name), - # init=mx.init.Constant(0.0), attr={'__lr_mult__': '2.0', '__wd_mult__': str(0.0)}) - # shared_vars.append( [shared_weight, shared_bias] ) - #else: - # shared_vars.append( [None, None] ) - # shared_vars.append( [None, None] ) - #if config.SHARE_WEIGHT_LANDMARK: - # _name = 'face_rpn_landmark_pred_share' - # shared_weight = mx.symbol.Variable(name="{}_weight".format(_name), - # init=mx.init.Normal(0.01), attr={'__lr_mult__': '1.0'}) - # shared_bias = mx.symbol.Variable(name="{}_bias".format(_name), - # init=mx.init.Constant(0.0), attr={'__lr_mult__': '2.0', '__wd_mult__': str(0.0)}) - # shared_vars.append( [shared_weight, shared_bias] ) - #else: - # shared_vars.append( [None, None] ) - - #for stride in config.RPN_FEAT_STRIDE: - # ret = get_out(conv_fpn_feat, 'face', stride, config.FACE_LANDMARK, lr_mult=1.0, shared_vars = shared_vars) - # ret_group += ret - # if config.HEAD_BOX: - # ret = get_out(conv_fpn_feat, 'head', stride, False, lr_mult=0.5) - # ret_group += ret - - #return mx.sym.Group(ret_group) - - diff --git a/RetinaFace/rcnn/symbol/symbol_resnet.py b/RetinaFace/rcnn/symbol/symbol_resnet.py deleted file mode 100644 index 8df5f1a..0000000 --- a/RetinaFace/rcnn/symbol/symbol_resnet.py +++ /dev/null @@ -1,423 +0,0 @@ -import mxnet as mx -import mxnet.ndarray as nd -import mxnet.gluon as gluon -import mxnet.gluon.nn as nn -import mxnet.autograd as ag -import numpy as np -from rcnn.config import config -from rcnn.PY_OP import rpn_fpn_ohem3 -from rcnn.symbol.symbol_common import get_sym_train - -def conv_only(from_layer, name, num_filter, kernel=(1,1), pad=(0,0), \ - stride=(1,1), bias_wd_mult=0.0): - weight = mx.symbol.Variable(name="{}_weight".format(name), - init=mx.init.Normal(0.01), attr={'__lr_mult__': '1.0'}) - bias = mx.symbol.Variable(name="{}_bias".format(name), - init=mx.init.Constant(0.0), attr={'__lr_mult__': '2.0', '__wd_mult__': str(bias_wd_mult)}) - conv = mx.symbol.Convolution(data=from_layer, kernel=kernel, pad=pad, \ - stride=stride, num_filter=num_filter, name="{}".format(name), weight = weight, bias=bias) - return conv - -def conv_act_layer_dw(from_layer, name, num_filter, kernel=(1,1), pad=(0,0), \ - stride=(1,1), act_type="relu", bias_wd_mult=0.0): - assert kernel[0]==3 - weight = mx.symbol.Variable(name="{}_weight".format(name), - init=mx.init.Normal(0.01), attr={'__lr_mult__': '1.0'}) - bias = mx.symbol.Variable(name="{}_bias".format(name), - init=mx.init.Constant(0.0), attr={'__lr_mult__': '2.0', '__wd_mult__': str(bias_wd_mult)}) - conv = mx.symbol.Convolution(data=from_layer, kernel=kernel, pad=pad, \ - stride=stride, num_filter=num_filter, num_group=num_filter, name="{}".format(name), weight=weight, bias=bias) - conv = mx.sym.BatchNorm(data=conv, fix_gamma=False, eps=2e-5, momentum=0.9, name=name + '_bn') - if len(act_type)>0: - relu = mx.symbol.Activation(data=conv, act_type=act_type, \ - name="{}_{}".format(name, act_type)) - else: - relu = conv - return relu - -def conv_act_layer(from_layer, name, num_filter, kernel=(1,1), pad=(0,0), \ - stride=(1,1), act_type="relu", bias_wd_mult=0.0, separable=False, filter_in = -1): - - separable = False - if separable: - assert kernel[0]==3 - if not separable: - weight = mx.symbol.Variable(name="{}_weight".format(name), - init=mx.init.Normal(0.01), attr={'__lr_mult__': '1.0'}) - bias = mx.symbol.Variable(name="{}_bias".format(name), - init=mx.init.Constant(0.0), attr={'__lr_mult__': '2.0', '__wd_mult__': str(bias_wd_mult)}) - conv = mx.symbol.Convolution(data=from_layer, kernel=kernel, pad=pad, \ - stride=stride, num_filter=num_filter, name="{}".format(name), weight=weight, bias=bias) - conv = mx.sym.BatchNorm(data=conv, fix_gamma=False, eps=2e-5, momentum=0.9, name=name + '_bn') - else: - if filter_in<0: - filter_in = num_filter - conv = mx.symbol.Convolution(data=from_layer, kernel=kernel, pad=pad, \ - stride=stride, num_filter=filter_in, num_group=filter_in, name="{}_sep".format(name)) - conv = mx.sym.BatchNorm(data=conv, fix_gamma=False, eps=2e-5, momentum=0.9, name=name + '_sep_bn') - conv = mx.symbol.Activation(data=conv, act_type='relu', \ - name="{}_sep_bn_relu".format(name)) - conv = mx.symbol.Convolution(data=conv, kernel=(1,1), pad=(0,0), \ - stride=(1,1), num_filter=num_filter, name="{}".format(name)) - conv = mx.sym.BatchNorm(data=conv, fix_gamma=False, eps=2e-5, momentum=0.9, name=name + '_bn') - if len(act_type)>0: - relu = mx.symbol.Activation(data=conv, act_type=act_type, \ - name="{}_{}".format(name, act_type)) - else: - relu = conv - return relu - -def ssh_context_module(body, num_filter, filter_in, name): - conv_dimred = conv_act_layer(body, name+'_conv1', - num_filter, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='relu', separable=True, filter_in = filter_in) - conv5x5 = conv_act_layer(conv_dimred, name+'_conv2', - num_filter, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='', separable=True) - conv7x7_1 = conv_act_layer(conv_dimred, name+'_conv3_1', - num_filter, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='relu', separable=True) - conv7x7 = conv_act_layer(conv7x7_1, name+'_conv3_2', - num_filter, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='', separable=True) - return (conv5x5, conv7x7) - -def conv_deformable(net, num_filter, num_group=1, act_type='relu',name=''): - f = num_group*18 - conv_offset = mx.symbol.Convolution(name=name+'_conv_offset', data = net, - num_filter=f, pad=(1, 1), kernel=(3, 3), stride=(1, 1)) - net = mx.contrib.symbol.DeformableConvolution(name=name+"_conv", data=net, offset=conv_offset, - num_filter=num_filter, pad=(1,1), kernel=(3, 3), num_deformable_group=num_group, stride=(1, 1), no_bias=False) - net = mx.sym.BatchNorm(data=net, fix_gamma=False, eps=2e-5, momentum=0.9, name=name + '_bn') - if len(act_type)>0: - net = mx.symbol.Activation(data=net, act_type=act_type, name=name+'_act') - return net - -def ssh_detection_module(body, num_filter, filter_in, name): - conv3x3 = conv_act_layer(body, name+'_conv1', - num_filter, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='', separable=True, filter_in=filter_in) - conv5x5, conv7x7 = ssh_context_module(body, num_filter//2, filter_in, name+'_context') - ret = mx.sym.concat(*[conv3x3, conv5x5, conv7x7], dim=1, name = name+'_concat') - ret = mx.symbol.Activation(data=ret, act_type='relu', name=name+'_concat_relu') - if config.USE_DCN==1: - ret = conv_deformable(ret, num_filter = num_filter*2, name = name+'_concat_dcn') - elif config.USE_DCN==2: - ret = conv_deformable2(ret, num_filter = num_filter*2, name = name+'_concat_dcn') - return ret - - -def get_resnet_conv(data, sym): - all_layers = sym.get_internals() - isize = 640 - _, out_shape, _ = all_layers.infer_shape(data = (1,3,isize,isize)) - last_entry = None - c1 = None - c2 = None - c3 = None - #print(len(all_layers), len(out_shape)) - #print(all_layers.__class__) - outputs = all_layers.list_outputs() - #print(outputs.__class__, len(outputs)) - count = len(outputs) - for i in range(count): - name = outputs[i] - shape = out_shape[i] - if not name.endswith('_output'): - continue - if len(shape)!=4: - continue - print(name, shape) - if c1 is None and shape[2]==isize//16: - cname = last_entry[0] - print('c1', last_entry) - c1 = all_layers[cname] - if c2 is None and shape[2]==isize//32: - cname = last_entry[0] - print('c2', last_entry) - c2 = all_layers[cname] - if shape[2]==isize//32: - c3 = all_layers[name] - print('c3', name, shape) - - last_entry = (name, shape) - - c1_filter = -1 - c2_filter = -1 - c3_filter = -1 - - F1 = 256 - F2 = 256 - _bwm = 1.0 - if config.NET_MODE==0: - c1_lateral = conv_act_layer(c1, 'ssh_m1_red_conv', - F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - c2_lateral = conv_act_layer(c2, 'ssh_m2_red_conv', - F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - #conv5_128_up = mx.symbol.Deconvolution(data=conv5_128, num_filter=F2, kernel=(4,4), stride=(2, 2), pad=(1,1), - # num_group = F2, no_bias = True, attr={'__lr_mult__': '0.0', '__wd_mult__': '0.0'}, - # name='ssh_m2_red_upsampling') - c2_up = mx.symbol.UpSampling(c2_lateral, scale=2, sample_type='nearest', workspace=512, name='ssh_m2_red_up', num_args=1) - #conv4_128 = mx.symbol.Crop(*[conv4_128, conv5_128_up]) - c2_up = mx.symbol.Crop(*[c2_up, c1_lateral]) - - c1 = c1_lateral+c2_up - - c1 = conv_act_layer(c1, 'ssh_m1_conv', - F2, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - m1 = ssh_detection_module(c1, F2, F2, 'ssh_m1_det') - m2 = ssh_detection_module(c2, F1, c2_filter, 'ssh_m2_det') - m3 = ssh_detection_module(c3, F1, c3_filter, 'ssh_m3_det') - elif config.NET_MODE==1: - c3_lateral = conv_act_layer(c3, 'ssh_c3_lateral', - F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - c3_up = mx.symbol.UpSampling(c3_lateral, scale=2, sample_type='nearest', workspace=512, name='ssh_c3_up', num_args=1) - c2_lateral = conv_act_layer(c2, 'ssh_c2_lateral', - F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - c3_up = mx.symbol.Crop(*[c3_up, c2_lateral]) - c2 = c2_lateral+c3_up - c2 = conv_act_layer(c2, 'ssh_c2_aggr', - F2, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - c1_lateral = conv_act_layer(c1, 'ssh_m1_red_conv', - F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - c2_up = mx.symbol.UpSampling(c2, scale=2, sample_type='nearest', workspace=512, name='ssh_m2_red_up', num_args=1) - #conv4_128 = mx.symbol.Crop(*[conv4_128, conv5_128_up]) - c2_up = mx.symbol.Crop(*[c2_up, c1_lateral]) - - c1 = c1_lateral+c2_up - - c1 = conv_act_layer(c1, 'ssh_m1_conv', - F2, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - m1 = ssh_detection_module(c1, F2, F2, 'ssh_m1_det') - m2 = ssh_detection_module(c2, F1, c2_filter, 'ssh_m2_det') - m3 = ssh_detection_module(c3, F1, c3_filter, 'ssh_m3_det') - elif config.NET_MODE==2: - n1 = ssh_detection_module(c1, F2, F2, 'ssh_n1_det') - n2 = ssh_detection_module(c2, F1, c2_filter, 'ssh_n2_det') - n3 = ssh_detection_module(c3, F1, c3_filter, 'ssh_n3_det') - c3 = conv_act_layer(c3, 'ssh_c3_lateral', - F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - c3_up = mx.symbol.UpSampling(c3, scale=2, sample_type='nearest', workspace=512, name='ssh_c3_up', num_args=1) - c2_lateral = conv_act_layer(c2, 'ssh_c2_lateral', - F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - c3_up = mx.symbol.Crop(*[c3_up, c2_lateral]) - c2 = c2_lateral+c3_up - c2 = conv_act_layer(c2, 'ssh_c2_aggr', - F2, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - c1_lateral = conv_act_layer(c1, 'ssh_m1_red_conv', - F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - c2_up = mx.symbol.UpSampling(c2, scale=2, sample_type='nearest', workspace=512, name='ssh_m2_red_up', num_args=1) - #conv4_128 = mx.symbol.Crop(*[conv4_128, conv5_128_up]) - c2_up = mx.symbol.Crop(*[c2_up, c1_lateral]) - c1 = c1_lateral+c2_up - c1 = conv_act_layer(c1, 'ssh_c1_aggr', - F2, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - - m1 = ssh_detection_module(c1, F2, F2, 'ssh_m1_det') - m2 = ssh_detection_module(c2, F1, c2_filter, 'ssh_m2_det') - m3 = ssh_detection_module(c3, F1, c3_filter, 'ssh_m3_det') - elif config.NET_MODE==3: - #c3 = conv_act_layer(c3, 'ssh_c3_lateral', - # F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - c3 = ssh_detection_module(c3, F2//2, c3_filter, 'ssh_c3_lateral') - c3_up = mx.symbol.UpSampling(c3, scale=2, sample_type='nearest', workspace=512, name='ssh_c3_up', num_args=1) - #c2_lateral = conv_act_layer(c2, 'ssh_c2_lateral', - # F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - c2_lateral = ssh_detection_module(c2, F2//2, c2_filter, 'ssh_c2_lateral') - c3_up = mx.symbol.Crop(*[c3_up, c2_lateral]) - c2 = c2_lateral+c3_up - c2 = conv_act_layer(c2, 'ssh_c2_aggr', - F2, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - #c1_lateral = conv_act_layer(c1, 'ssh_m1_red_conv', - # F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - c1_lateral = ssh_detection_module(c1, F2//2, c1_filter, 'ssh_c1_lateral') - c2_up = mx.symbol.UpSampling(c2, scale=2, sample_type='nearest', workspace=512, name='ssh_m2_red_up', num_args=1) - #conv4_128 = mx.symbol.Crop(*[conv4_128, conv5_128_up]) - c2_up = mx.symbol.Crop(*[c2_up, c1_lateral]) - c1 = c1_lateral+c2_up - c1 = conv_act_layer(c1, 'ssh_c1_aggr', - F2, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - - m1 = ssh_detection_module(c1, F2, F2, 'ssh_m1_det') - m2 = ssh_detection_module(c2, F1, c2_filter, 'ssh_m2_det') - m3 = ssh_detection_module(c3, F1, c3_filter, 'ssh_m3_det') - elif config.NET_MODE==4: - c3 = conv_act_layer(c3, 'ssh_c3_lateral', - F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - c3_up = mx.symbol.UpSampling(c3, scale=2, sample_type='nearest', workspace=512, name='ssh_c3_up', num_args=1) - c2_lateral = conv_act_layer(c2, 'ssh_c2_lateral', - F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - c3_up = mx.symbol.Crop(*[c3_up, c2_lateral]) - c2 = c2_lateral+c3_up - c2 = conv_act_layer(c2, 'ssh_c2_aggr', - F2, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - c1_lateral = conv_act_layer(c1, 'ssh_m1_red_conv', - F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - c2_up = mx.symbol.UpSampling(c2, scale=2, sample_type='nearest', workspace=512, name='ssh_m2_red_up', num_args=1) - #conv4_128 = mx.symbol.Crop(*[conv4_128, conv5_128_up]) - c2_up = mx.symbol.Crop(*[c2_up, c1_lateral]) - c1 = c1_lateral+c2_up - c1 = conv_act_layer(c1, 'ssh_c1_aggr', - F2, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - - m1 = ssh_detection_module(c1, F2//2, F2, 'ssh_m1_det') - m2 = ssh_detection_module(c2, F1//2, c2_filter, 'ssh_m2_det') - m3 = ssh_detection_module(c3, F1//2, c3_filter, 'ssh_m3_det') - elif config.NET_MODE==5: - c3 = conv_act_layer_dw(c3, 'ssh_c3_lateral_m', - F2, kernel=(3,3), pad=(1,1), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - c3 = conv_act_layer(c3, 'ssh_c3_lateral', - F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - c3_up = mx.symbol.UpSampling(c3, scale=2, sample_type='nearest', workspace=512, name='ssh_c3_up', num_args=1) - c2 = conv_act_layer_dw(c2, 'ssh_c2_lateral_m', - F2, kernel=(3,3), pad=(1,1), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - c2_lateral = conv_act_layer(c2, 'ssh_c2_lateral', - F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - c3_up = mx.symbol.Crop(*[c3_up, c2_lateral]) - c2 = c2_lateral+c3_up - c2 = conv_act_layer(c2, 'ssh_c2_aggr', - F2, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - c1 = conv_act_layer_dw(c1, 'ssh_c1_lateral_m', - F2, kernel=(3,3), pad=(1,1), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - c1_lateral = conv_act_layer(c1, 'ssh_m1_red_conv', - F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - c2_up = mx.symbol.UpSampling(c2, scale=2, sample_type='nearest', workspace=512, name='ssh_m2_red_up', num_args=1) - #conv4_128 = mx.symbol.Crop(*[conv4_128, conv5_128_up]) - c2_up = mx.symbol.Crop(*[c2_up, c1_lateral]) - c1 = c1_lateral+c2_up - c1 = conv_act_layer(c1, 'ssh_c1_aggr', - F2, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - - m1 = ssh_detection_module(c1, F2, F2, 'ssh_m1_det') - m2 = ssh_detection_module(c2, F1, c2_filter, 'ssh_m2_det') - m3 = ssh_detection_module(c3, F1, c3_filter, 'ssh_m3_det') - - return {8: m1, 16:m2, 32: m3}, {8: n1, 16:n2, 32: n3} - -def get_out(conv_fpn_feat, prefix, stride, landmark=False, lr_mult=1.0): - A = config.NUM_ANCHORS - bbox_pred_len = 4 - landmark_pred_len = 10 - if config.USE_BLUR: - bbox_pred_len = 5 - if config.USE_OCCLUSION: - landmark_pred_len = 15 - ret_group = [] - num_anchors = config.RPN_ANCHOR_CFG[str(stride)]['NUM_ANCHORS'] - label = mx.symbol.Variable(name='%s_label_stride%d'%(prefix,stride)) - bbox_target = mx.symbol.Variable(name='%s_bbox_target_stride%d'%(prefix,stride)) - bbox_weight = mx.symbol.Variable(name='%s_bbox_weight_stride%d'%(prefix,stride)) - if landmark: - landmark_target = mx.symbol.Variable(name='%s_landmark_target_stride%d'%(prefix,stride)) - landmark_weight = mx.symbol.Variable(name='%s_landmark_weight_stride%d'%(prefix,stride)) - rpn_relu = conv_fpn_feat[stride] - maxout_stat = 0 - if config.USE_MAXOUT>=1 and stride==config.RPN_FEAT_STRIDE[-1]: - maxout_stat = 1 - if config.USE_MAXOUT>=2 and stride!=config.RPN_FEAT_STRIDE[-1]: - maxout_stat = 2 - - if maxout_stat==0: - rpn_cls_score = conv_only(rpn_relu, '%s_rpn_cls_score_stride%d'%(prefix, stride), 2*num_anchors, - kernel=(1,1), pad=(0,0), stride=(1, 1)) - elif maxout_stat==1: - cls_list = [] - for a in range(num_anchors): - rpn_cls_score_bg = conv_only(rpn_relu, '%s_rpn_cls_score_stride%d_anchor%d_bg'%(prefix,stride,a), 3, - kernel=(1,1), pad=(0,0), stride=(1, 1)) - rpn_cls_score_bg = mx.sym.max(rpn_cls_score_bg, axis=1, keepdims=True) - cls_list.append(rpn_cls_score_bg) - rpn_cls_score_fg = conv_only(rpn_relu, '%s_rpn_cls_score_stride%d_anchor%d_fg'%(prefix,stride,a), 1, - kernel=(1,1), pad=(0,0), stride=(1, 1)) - cls_list.append(rpn_cls_score_fg) - rpn_cls_score = mx.sym.concat(*cls_list, dim=1, name='%s_rpn_cls_score_stride%d'%(prefix,stride)) - else: - cls_list = [] - for a in range(num_anchors): - rpn_cls_score_bg = conv_only(rpn_relu, '%s_rpn_cls_score_stride%d_anchor%d_bg'%(prefix,stride,a), 1, - kernel=(1,1), pad=(0,0), stride=(1, 1)) - cls_list.append(rpn_cls_score_bg) - rpn_cls_score_fg = conv_only(rpn_relu, '%s_rpn_cls_score_stride%d_anchor%d_fg'%(prefix,stride,a), 3, - kernel=(1,1), pad=(0,0), stride=(1, 1)) - rpn_cls_score_fg = mx.sym.max(rpn_cls_score_fg, axis=1, keepdims=True) - cls_list.append(rpn_cls_score_fg) - rpn_cls_score = mx.sym.concat(*cls_list, dim=1, name='%s_rpn_cls_score_stride%d'%(prefix,stride)) - - rpn_bbox_pred = conv_only(rpn_relu, '%s_rpn_bbox_pred_stride%d'%(prefix,stride), bbox_pred_len*num_anchors, - kernel=(1,1), pad=(0,0), stride=(1, 1)) - - # prepare rpn data - if not config.FBN: - rpn_cls_score_reshape = mx.symbol.Reshape(data=rpn_cls_score, - shape=(0, 2, -1), - name="%s_rpn_cls_score_reshape_stride%s" % (prefix,stride)) - else: - rpn_cls_score_reshape = mx.symbol.Reshape(data=rpn_cls_score, - shape=(0, 2, -1), - name="%s_rpn_cls_score_reshape_stride%s_pre" % (prefix,stride)) - rpn_cls_score_reshape = mx.symbol.BatchNorm(rpn_cls_score_reshape, fix_gamma=True, eps=2e-5, name="%s_rpn_cls_score_reshape_stride%s"%(prefix, stride)) - - rpn_bbox_pred_reshape = mx.symbol.Reshape(data=rpn_bbox_pred, - shape=(0, 0, -1), - name="%s_rpn_bbox_pred_reshape_stride%s" % (prefix,stride)) - if landmark: - rpn_landmark_pred = conv_only(rpn_relu, '%s_rpn_landmark_pred_stride%d'%(prefix,stride), landmark_pred_len*num_anchors, - kernel=(1,1), pad=(0,0), stride=(1, 1)) - rpn_landmark_pred_reshape = mx.symbol.Reshape(data=rpn_landmark_pred, - shape=(0, 0, -1), - name="%s_rpn_landmark_pred_reshape_stride%s" % (prefix,stride)) - - if config.TRAIN.RPN_ENABLE_OHEM>=2: - label, anchor_weight = mx.sym.Custom(op_type='rpn_fpn_ohem3', stride=int(stride), network=config.network, dataset=config.dataset, prefix=prefix, cls_score=rpn_cls_score_reshape, labels = label) - - _bbox_weight = mx.sym.tile(anchor_weight, (1,1,bbox_pred_len)) - _bbox_weight = _bbox_weight.reshape((0, -1, A * bbox_pred_len)).transpose((0,2,1)) - bbox_weight = mx.sym.elemwise_mul(bbox_weight, _bbox_weight, name='%s_bbox_weight_mul_stride%s'%(prefix,stride)) - - if landmark: - _landmark_weight = mx.sym.tile(anchor_weight, (1,1,landmark_pred_len)) - _landmark_weight = _landmark_weight.reshape((0, -1, A * landmark_pred_len)).transpose((0,2,1)) - landmark_weight = mx.sym.elemwise_mul(landmark_weight, _landmark_weight, name='%s_landmark_weight_mul_stride%s'%(prefix,stride)) - #if not config.FACE_LANDMARK: - # label, bbox_weight = mx.sym.Custom(op_type='rpn_fpn_ohem', stride=int(stride), cls_score=rpn_cls_score_reshape, bbox_weight = bbox_weight , labels = label) - #else: - # label, bbox_weight, landmark_weight = mx.sym.Custom(op_type='rpn_fpn_ohem2', stride=int(stride), cls_score=rpn_cls_score_reshape, bbox_weight = bbox_weight, landmark_weight=landmark_weight, labels = label) - #cls loss - rpn_cls_prob = mx.symbol.SoftmaxOutput(data=rpn_cls_score_reshape, - label=label, - multi_output=True, - normalization='valid', use_ignore=True, ignore_label=-1, - grad_scale = lr_mult, - name='%s_rpn_cls_prob_stride%d'%(prefix,stride)) - ret_group.append(rpn_cls_prob) - ret_group.append(mx.sym.BlockGrad(label)) - - #bbox loss - bbox_diff = rpn_bbox_pred_reshape-bbox_target - bbox_diff = bbox_diff * bbox_weight - rpn_bbox_loss_ = mx.symbol.smooth_l1(name='%s_rpn_bbox_loss_stride%d_'%(prefix,stride), scalar=3.0, data=bbox_diff) - rpn_bbox_loss = mx.sym.MakeLoss(name='%s_rpn_bbox_loss_stride%d'%(prefix,stride), data=rpn_bbox_loss_, grad_scale=1.0*lr_mult / (config.TRAIN.RPN_BATCH_SIZE)) - ret_group.append(rpn_bbox_loss) - ret_group.append(mx.sym.BlockGrad(bbox_weight)) - - #landmark loss - if landmark: - landmark_diff = rpn_landmark_pred_reshape-landmark_target - landmark_diff = landmark_diff * landmark_weight - rpn_landmark_loss_ = mx.symbol.smooth_l1(name='%s_rpn_landmark_loss_stride%d_'%(prefix,stride), scalar=3.0, data=landmark_diff) - rpn_landmark_loss = mx.sym.MakeLoss(name='%s_rpn_landmark_loss_stride%d'%(prefix,stride), data=rpn_landmark_loss_, grad_scale=0.5*lr_mult / (config.TRAIN.RPN_BATCH_SIZE)) - ret_group.append(rpn_landmark_loss) - ret_group.append(mx.sym.BlockGrad(landmark_weight)) - return ret_group - -def get_resnet_train(sym): - return get_sym_train(sym) - #data = mx.symbol.Variable(name="data") - ## shared convolutional layers - #conv_fpn_feat, conv_fpn_feat2 = get_resnet_conv(data, sym) - #ret_group = [] - #for stride in config.RPN_FEAT_STRIDE: - # ret = get_out(conv_fpn_feat, 'face', stride, config.FACE_LANDMARK, lr_mult=1.0) - # ret_group += ret - # if config.HEAD_BOX: - # ret = get_out(conv_fpn_feat2, 'head', stride, False, lr_mult=1.0) - # ret_group += ret - - #return mx.sym.Group(ret_group) - - diff --git a/RetinaFace/rcnn/symbol/symbol_ssh.py b/RetinaFace/rcnn/symbol/symbol_ssh.py deleted file mode 100644 index 31541ee..0000000 --- a/RetinaFace/rcnn/symbol/symbol_ssh.py +++ /dev/null @@ -1,365 +0,0 @@ -import mxnet as mx -import numpy as np -from rcnn.config import config -from rcnn.PY_OP import rpn_fpn_ohem3 -FPN = False -USE_DCN=False - -def conv_act_layer(from_layer, name, num_filter, kernel=(1,1), pad=(0,0), \ - stride=(1,1), act_type="relu", bias_wd_mult=0.0, dcn=False): - - weight = mx.symbol.Variable(name="{}_weight".format(name), - init=mx.init.Normal(0.01), attr={'__lr_mult__': '1.0'}) - bias = mx.symbol.Variable(name="{}_bias".format(name), - init=mx.init.Constant(0.0), attr={'__lr_mult__': '2.0', '__wd_mult__': str(bias_wd_mult)}) - if not dcn: - conv = mx.symbol.Convolution(data=from_layer, kernel=kernel, pad=pad, \ - stride=stride, num_filter=num_filter, name="{}".format(name), weight = weight, bias=bias) - else: - assert kernel[0]==3 and kernel[1]==3 - num_group = 1 - f = num_group*18 - offset_weight = mx.symbol.Variable(name="{}_offset_weight".format(name), - init=mx.init.Constant(0.0), attr={'__lr_mult__': '1.0'}) - offset_bias = mx.symbol.Variable(name="{}_offset_bias".format(name), - init=mx.init.Constant(0.0), attr={'__lr_mult__': '2.0', '__wd_mult__': str(bias_wd_mult)}) - conv_offset = mx.symbol.Convolution(name=name+'_offset', data = from_layer, weight=offset_weight, bias=offset_bias, - num_filter=f, pad=(1, 1), kernel=(3, 3), stride=(1, 1)) - conv = mx.contrib.symbol.DeformableConvolution(name=name, data=from_layer, offset=conv_offset, weight=weight, bias=bias, - num_filter=num_filter, pad=(1,1), kernel=(3, 3), num_deformable_group=num_group, stride=(1, 1), no_bias=False) - if len(act_type)>0: - relu = mx.symbol.Activation(data=conv, act_type=act_type, \ - name="{}_{}".format(name, act_type)) - else: - relu = conv - return relu - -def ssh_context_module(body, num_filters, name): - conv_dimred = conv_act_layer(body, name+'_conv1', - num_filters, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='relu', dcn=False) - conv5x5 = conv_act_layer(conv_dimred, name+'_conv2', - num_filters, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='', dcn=USE_DCN) - conv7x7_1 = conv_act_layer(conv_dimred, name+'_conv3_1', - num_filters, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='relu', dcn=False) - conv7x7 = conv_act_layer(conv7x7_1, name+'_conv3_2', - num_filters, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='', dcn=USE_DCN) - return (conv5x5, conv7x7) - -def ssh_detection_module(body, num_filters, name): - conv3x3 = conv_act_layer(body, name+'_conv1', - num_filters, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='', dcn=USE_DCN) - conv5x5, conv7x7 = ssh_context_module(body, num_filters//2, name+'_context') - ret = mx.sym.concat(*[conv3x3, conv5x5, conv7x7], dim=1, name = name+'_concat') - ret = mx.symbol.Activation(data=ret, act_type='relu', name=name+'_concat_relu') - return ret - -def conv_bn(input, filter, ksize, stride, padding, act_type='relu', name=''): - conv = mx.symbol.Convolution(data=input, kernel=(ksize,ksize), pad=(padding,padding), \ - stride=(stride,stride), num_filter=filter, name=name+"_conv") - ret = mx.sym.BatchNorm(data=conv, fix_gamma=False, eps=2e-5, momentum=0.9, name=name + '_bn') - if act_type is not None: - ret = mx.symbol.Activation(data=ret, act_type=act_type, \ - name="{}_{}".format(name, act_type)) - return ret - -def cpm(input, name): - # residual - branch1 = conv_bn(input, 1024, 1, 1, 0, act_type=None, name=name+"_branch1") - branch2a = conv_bn(input, 256, 1, 1, 0, act_type='relu', name=name+"_branch2a") - branch2b = conv_bn(branch2a, 256, 3, 1, 1, act_type='relu', name=name+"_branch2b") - branch2c = conv_bn(branch2b, 1024, 1, 1, 0, act_type=None, name=name+"_branch2c") - sum = branch1 + branch2c - rescomb = mx.symbol.Activation(data=sum, act_type='relu', name="%s_relu2"%(name)) - - ssh_out = ssh_detection_module(rescomb, 256, name=name+"_ssh") - return ssh_out - -def get_feat_down(conv_feat): - #P5 = mx.symbol.Convolution(data=conv_feat[0], kernel=(1, 1), num_filter=256, name="P5_lateral") - P5 = conv_act_layer(conv_feat[0], 'P5_lateral', - 256, kernel=(1,1), pad=(0,0), stride=(1, 1), act_type='relu') - - # P5 2x upsampling + C4 = P4 - P5_up = mx.symbol.UpSampling(P5, scale=2, sample_type='nearest', workspace=512, name='P5_upsampling', num_args=1) - #P4_la = mx.symbol.Convolution(data=conv_feat[1], kernel=(1, 1), num_filter=256, name="P4_lateral") - P4_la = conv_act_layer(conv_feat[1], 'P4_lateral', - 256, kernel=(1,1), pad=(0,0), stride=(1, 1), act_type='relu') - P5_clip = mx.symbol.Crop(*[P5_up, P4_la], name="P4_clip") - P4 = mx.sym.ElementWiseSum(*[P5_clip, P4_la], name="P4_sum") - #P4 = mx.symbol.Convolution(data=P4, kernel=(3, 3), pad=(1, 1), num_filter=256, name="P4_aggregate") - P4 = conv_act_layer(P4, 'P4_aggregate', - 256, kernel=(3,3), pad=(1,1), stride=(1, 1), act_type='relu') - - # P4 2x upsampling + C3 = P3 - P4_up = mx.symbol.UpSampling(P4, scale=2, sample_type='nearest', workspace=512, name='P4_upsampling', num_args=1) - #P3_la = mx.symbol.Convolution(data=conv_feat[2], kernel=(1, 1), num_filter=256, name="P3_lateral") - P3_la = conv_act_layer(conv_feat[2], 'P3_lateral', - 256, kernel=(1,1), pad=(0,0), stride=(1, 1), act_type='relu') - P4_clip = mx.symbol.Crop(*[P4_up, P3_la], name="P3_clip") - P3 = mx.sym.ElementWiseSum(*[P4_clip, P3_la], name="P3_sum") - #P3 = mx.symbol.Convolution(data=P3, kernel=(3, 3), pad=(1, 1), num_filter=256, name="P3_aggregate") - P3 = conv_act_layer(P3, 'P3_aggregate', - 256, kernel=(3,3), pad=(1,1), stride=(1, 1), act_type='relu') - - return P3, P4, P5 - -def get_ssh_conv(data): - """ - shared convolutional layers - :param data: Symbol - :return: Symbol - """ - # group 1 - #conv1_1 = mx.symbol.Convolution( - # data=data, kernel=(3, 3), pad=(1, 1), num_filter=64, workspace=2048, name="conv1_1") - #relu1_1 = mx.symbol.Activation(data=conv1_1, act_type="relu", name="relu1_1") - relu1_1 = conv_act_layer(data, 'conv1_1', - 64, kernel=(3,3), pad=(1,1), stride=(1, 1), act_type='relu') - #conv1_2 = mx.symbol.Convolution( - # data=relu1_1, kernel=(3, 3), pad=(1, 1), num_filter=64, workspace=2048, name="conv1_2") - #relu1_2 = mx.symbol.Activation(data=conv1_2, act_type="relu", name="relu1_2") - relu1_2 = conv_act_layer(relu1_1, 'conv1_2', - 64, kernel=(3,3), pad=(1,1), stride=(1, 1), act_type='relu') - pool1 = mx.symbol.Pooling( - data=relu1_2, pool_type="max", kernel=(2, 2), stride=(2, 2), name="pool1") - # group 2 - #conv2_1 = mx.symbol.Convolution( - # data=pool1, kernel=(3, 3), pad=(1, 1), num_filter=128, workspace=2048, name="conv2_1") - #relu2_1 = mx.symbol.Activation(data=conv2_1, act_type="relu", name="relu2_1") - relu2_1 = conv_act_layer(pool1, 'conv2_1', - 128, kernel=(3,3), pad=(1,1), stride=(1, 1), act_type='relu') - #conv2_2 = mx.symbol.Convolution( - # data=relu2_1, kernel=(3, 3), pad=(1, 1), num_filter=128, workspace=2048, name="conv2_2") - #relu2_2 = mx.symbol.Activation(data=conv2_2, act_type="relu", name="relu2_2") - relu2_2 = conv_act_layer(relu2_1, 'conv2_2', - 128, kernel=(3,3), pad=(1,1), stride=(1, 1), act_type='relu') - pool2 = mx.symbol.Pooling( - data=relu2_2, pool_type="max", kernel=(2, 2), stride=(2, 2), name="pool2") - # group 3 - #conv3_1 = mx.symbol.Convolution( - # data=pool2, kernel=(3, 3), pad=(1, 1), num_filter=256, workspace=2048, name="conv3_1") - #relu3_1 = mx.symbol.Activation(data=conv3_1, act_type="relu", name="relu3_1") - relu3_1 = conv_act_layer(pool2, 'conv3_1', - 256, kernel=(3,3), pad=(1,1), stride=(1, 1), act_type='relu') - #conv3_2 = mx.symbol.Convolution( - # data=relu3_1, kernel=(3, 3), pad=(1, 1), num_filter=256, workspace=2048, name="conv3_2") - #relu3_2 = mx.symbol.Activation(data=conv3_2, act_type="relu", name="relu3_2") - relu3_2 = conv_act_layer(relu3_1, 'conv3_2', - 256, kernel=(3,3), pad=(1,1), stride=(1, 1), act_type='relu') - #conv3_3 = mx.symbol.Convolution( - # data=relu3_2, kernel=(3, 3), pad=(1, 1), num_filter=256, workspace=2048, name="conv3_3") - #relu3_3 = mx.symbol.Activation(data=conv3_3, act_type="relu", name="relu3_3") - relu3_3 = conv_act_layer(relu3_2, 'conv3_3', - 256, kernel=(3,3), pad=(1,1), stride=(1, 1), act_type='relu') - pool3 = mx.symbol.Pooling( - data=relu3_3, pool_type="max", kernel=(2, 2), stride=(2, 2), name="pool3") - # group 4 - #conv4_1 = mx.symbol.Convolution( - # data=pool3, kernel=(3, 3), pad=(1, 1), num_filter=512, workspace=2048, name="conv4_1") - #relu4_1 = mx.symbol.Activation(data=conv4_1, act_type="relu", name="relu4_1") - relu4_1 = conv_act_layer(pool3, 'conv4_1', - 512, kernel=(3,3), pad=(1,1), stride=(1, 1), act_type='relu') - #conv4_2 = mx.symbol.Convolution( - # data=relu4_1, kernel=(3, 3), pad=(1, 1), num_filter=512, workspace=2048, name="conv4_2") - #relu4_2 = mx.symbol.Activation(data=conv4_2, act_type="relu", name="relu4_2") - relu4_2 = conv_act_layer(relu4_1, 'conv4_2', - 512, kernel=(3,3), pad=(1,1), stride=(1, 1), act_type='relu') - #conv4_3 = mx.symbol.Convolution( - # data=relu4_2, kernel=(3, 3), pad=(1, 1), num_filter=512, workspace=2048, name="conv4_3") - #relu4_3 = mx.symbol.Activation(data=conv4_3, act_type="relu", name="relu4_3") - relu4_3 = conv_act_layer(relu4_2, 'conv4_3', - 512, kernel=(3,3), pad=(1,1), stride=(1, 1), act_type='relu') - pool4 = mx.symbol.Pooling( - data=relu4_3, pool_type="max", kernel=(2, 2), stride=(2, 2), name="pool4") - # group 5 - #conv5_1 = mx.symbol.Convolution( - # data=pool4, kernel=(3, 3), pad=(1, 1), num_filter=512, workspace=2048, name="conv5_1") - #relu5_1 = mx.symbol.Activation(data=conv5_1, act_type="relu", name="relu5_1") - relu5_1 = conv_act_layer(pool4, 'conv5_1', - 512, kernel=(3,3), pad=(1,1), stride=(1, 1), act_type='relu') - #conv5_2 = mx.symbol.Convolution( - # data=relu5_1, kernel=(3, 3), pad=(1, 1), num_filter=512, workspace=2048, name="conv5_2") - #relu5_2 = mx.symbol.Activation(data=conv5_2, act_type="relu", name="relu5_2") - relu5_2 = conv_act_layer(relu5_1, 'conv5_2', - 512, kernel=(3,3), pad=(1,1), stride=(1, 1), act_type='relu') - #conv5_3 = mx.symbol.Convolution( - # data=relu5_2, kernel=(3, 3), pad=(1, 1), num_filter=512, workspace=2048, name="conv5_3") - #relu5_3 = mx.symbol.Activation(data=conv5_3, act_type="relu", name="relu5_3") - relu5_3 = conv_act_layer(relu5_2, 'conv5_3', - 512, kernel=(3,3), pad=(1,1), stride=(1, 1), act_type='relu') - m3_pool = mx.sym.Pooling(data=relu5_3, kernel=(2, 2), stride=(2,2), pad=(0,0), pool_type='max') - if config.SSH_MODE<=5: - #if FPN: - # relu4_3, relu5_3, m3_pool = get_feat_down([m3_pool, relu5_3, relu4_3]) - - F1 = 256 - F2 = 128 - if config.SSH_MODE==1: - F2 = 256 - _bwm = 1.0 - conv4_128 = conv_act_layer(relu4_3, 'ssh_m1_red_conv', - F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - conv5_128 = conv_act_layer(relu5_3, 'ssh_m2_red_conv', - F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - conv5_128_up = mx.symbol.Deconvolution(data=conv5_128, num_filter=F2, kernel=(4,4), stride=(2, 2), pad=(1,1), - num_group = F2, no_bias = True, attr={'__lr_mult__': '0.0', '__wd_mult__': '0.0'}, - name='ssh_m2_red_upsampling') - #conv5_128_up = mx.symbol.UpSampling(conv5_128, scale=2, sample_type='nearest', workspace=512, name='ssh_m2_red_up', num_args=1) - conv4_128 = mx.symbol.Crop(*[conv4_128, conv5_128_up]) - #conv5_128_up = mx.symbol.Crop(*[conv5_128_up, conv4_128]) - - conv_sum = conv4_128+conv5_128_up - #conv_sum = conv_1x1 - - m1_conv = conv_act_layer(conv_sum, 'ssh_m1_conv', - F2, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - m1 = ssh_detection_module(m1_conv, F2, 'ssh_m1_det') - m2 = ssh_detection_module(relu5_3, F1, 'ssh_m2_det') - m3 = ssh_detection_module(m3_pool, F1, 'ssh_m3_det') - return {8: m1, 16:m2, 32: m3} - else: - F1 = 256 - F2 = 256 - _bwm = 1.0 - conv4_128 = conv_act_layer(relu4_3, 'ssh_m1_red_conv', - F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - conv5_128 = conv_act_layer(relu5_3, 'ssh_m2_red_conv', - F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - conv5_128_up = mx.symbol.Deconvolution(data=conv5_128, num_filter=F2, kernel=(4,4), stride=(2, 2), pad=(1,1), - num_group = F2, no_bias = True, attr={'__lr_mult__': '0.0', '__wd_mult__': '0.0'}, - name='ssh_m2_red_upsampling') - #conv5_128_up = mx.symbol.UpSampling(conv5_128, scale=2, sample_type='nearest', workspace=512, name='ssh_m2_red_up', num_args=1) - conv4_128 = mx.symbol.Crop(*[conv4_128, conv5_128_up]) - #conv5_128_up = mx.symbol.Crop(*[conv5_128_up, conv4_128]) - - conv_sum = conv4_128+conv5_128_up - m1_conv = conv_act_layer(conv_sum, 'ssh_m1_conv', - F2, kernel=(3, 3), pad=(1, 1), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) - m1 = cpm(m1_conv, 'ssh_m1_det') - m2 = cpm(relu5_3, 'ssh_m2_det') - m3 = cpm(m3_pool, 'ssh_m3_det') - return {8: m1, 16:m2, 32: m3} - -def get_out(conv_fpn_feat, prefix, stride, landmark=False, lr_mult=1.0): - A = config.NUM_ANCHORS - ret_group = [] - num_anchors = config.RPN_ANCHOR_CFG[str(stride)]['NUM_ANCHORS'] - label = mx.symbol.Variable(name='%s_label_stride%d'%(prefix,stride)) - bbox_target = mx.symbol.Variable(name='%s_bbox_target_stride%d'%(prefix,stride)) - bbox_weight = mx.symbol.Variable(name='%s_bbox_weight_stride%d'%(prefix,stride)) - if landmark: - landmark_target = mx.symbol.Variable(name='%s_landmark_target_stride%d'%(prefix,stride)) - landmark_weight = mx.symbol.Variable(name='%s_landmark_weight_stride%d'%(prefix,stride)) - rpn_relu = conv_fpn_feat[stride] - maxout_stat = 0 - if config.USE_MAXOUT>=1 and stride==config.RPN_FEAT_STRIDE[-1]: - maxout_stat = 1 - if config.USE_MAXOUT>=2 and stride!=config.RPN_FEAT_STRIDE[-1]: - maxout_stat = 2 - - if maxout_stat==0: - rpn_cls_score = conv_act_layer(rpn_relu, '%s_rpn_cls_score_stride%d'%(prefix, stride), 2*num_anchors, - kernel=(1,1), pad=(0,0), stride=(1, 1), act_type='') - elif maxout_stat==1: - cls_list = [] - for a in range(num_anchors): - rpn_cls_score_bg = conv_act_layer(rpn_relu, '%s_rpn_cls_score_stride%d_anchor%d_bg'%(prefix,stride,a), 3, - kernel=(1,1), pad=(0,0), stride=(1, 1), act_type='') - rpn_cls_score_bg = mx.sym.max(rpn_cls_score_bg, axis=1, keepdims=True) - cls_list.append(rpn_cls_score_bg) - rpn_cls_score_fg = conv_act_layer(rpn_relu, '%s_rpn_cls_score_stride%d_anchor%d_fg'%(prefix,stride,a), 1, - kernel=(1,1), pad=(0,0), stride=(1, 1), act_type='') - cls_list.append(rpn_cls_score_fg) - rpn_cls_score = mx.sym.concat(*cls_list, dim=1, name='%s_rpn_cls_score_stride%d'%(prefix,stride)) - else: - cls_list = [] - for a in range(num_anchors): - rpn_cls_score_bg = conv_act_layer(rpn_relu, '%s_rpn_cls_score_stride%d_anchor%d_bg'%(prefix,stride,a), 1, - kernel=(1,1), pad=(0,0), stride=(1, 1), act_type='') - cls_list.append(rpn_cls_score_bg) - rpn_cls_score_fg = conv_act_layer(rpn_relu, '%s_rpn_cls_score_stride%d_anchor%d_fg'%(prefix,stride,a), 3, - kernel=(1,1), pad=(0,0), stride=(1, 1), act_type='') - rpn_cls_score_fg = mx.sym.max(rpn_cls_score_fg, axis=1, keepdims=True) - cls_list.append(rpn_cls_score_fg) - rpn_cls_score = mx.sym.concat(*cls_list, dim=1, name='%s_rpn_cls_score_stride%d'%(prefix,stride)) - - rpn_bbox_pred = conv_act_layer(rpn_relu, '%s_rpn_bbox_pred_stride%d'%(prefix,stride), 4*num_anchors, - kernel=(1,1), pad=(0,0), stride=(1, 1), act_type='') - - # prepare rpn data - rpn_cls_score_reshape = mx.symbol.Reshape(data=rpn_cls_score, - shape=(0, 2, -1), - name="%s_rpn_cls_score_reshape_stride%s" % (prefix,stride)) - rpn_bbox_pred_reshape = mx.symbol.Reshape(data=rpn_bbox_pred, - shape=(0, 0, -1), - name="%s_rpn_bbox_pred_reshape_stride%s" % (prefix,stride)) - if landmark: - rpn_landmark_pred = conv_act_layer(rpn_relu, '%s_rpn_landmark_pred_stride%d'%(prefix,stride), 10*num_anchors, - kernel=(1,1), pad=(0,0), stride=(1, 1), act_type='') - rpn_landmark_pred_reshape = mx.symbol.Reshape(data=rpn_landmark_pred, - shape=(0, 0, -1), - name="%s_rpn_landmark_pred_reshape_stride%s" % (prefix,stride)) - - if config.TRAIN.RPN_ENABLE_OHEM>=2: - label, anchor_weight = mx.sym.Custom(op_type='rpn_fpn_ohem3', stride=int(stride), network=config.network, dataset=config.dataset, prefix=prefix, cls_score=rpn_cls_score_reshape, labels = label) - - _bbox_weight = mx.sym.tile(anchor_weight, (1,1,4)) - _bbox_weight = _bbox_weight.reshape((0, -1, A * 4)).transpose((0,2,1)) - bbox_weight = mx.sym.elemwise_mul(bbox_weight, _bbox_weight, name='%s_bbox_weight_mul_stride%s'%(prefix,stride)) - - if landmark: - _landmark_weight = mx.sym.tile(anchor_weight, (1,1,10)) - _landmark_weight = _landmark_weight.reshape((0, -1, A * 10)).transpose((0,2,1)) - landmark_weight = mx.sym.elemwise_mul(landmark_weight, _landmark_weight, name='%s_landmark_weight_mul_stride%s'%(prefix,stride)) - #if not config.FACE_LANDMARK: - # label, bbox_weight = mx.sym.Custom(op_type='rpn_fpn_ohem', stride=int(stride), cls_score=rpn_cls_score_reshape, bbox_weight = bbox_weight , labels = label) - #else: - # label, bbox_weight, landmark_weight = mx.sym.Custom(op_type='rpn_fpn_ohem2', stride=int(stride), cls_score=rpn_cls_score_reshape, bbox_weight = bbox_weight, landmark_weight=landmark_weight, labels = label) - #cls loss - rpn_cls_prob = mx.symbol.SoftmaxOutput(data=rpn_cls_score_reshape, - label=label, - multi_output=True, - normalization='valid', use_ignore=True, ignore_label=-1, - grad_scale = lr_mult, - name='%s_rpn_cls_prob_stride%d'%(prefix,stride)) - ret_group.append(rpn_cls_prob) - ret_group.append(mx.sym.BlockGrad(label)) - - #bbox loss - bbox_diff = rpn_bbox_pred_reshape-bbox_target - bbox_diff = bbox_diff * bbox_weight - rpn_bbox_loss_ = mx.symbol.smooth_l1(name='%s_rpn_bbox_loss_stride%d_'%(prefix,stride), scalar=3.0, data=bbox_diff) - rpn_bbox_loss = mx.sym.MakeLoss(name='%s_rpn_bbox_loss_stride%d'%(prefix,stride), data=rpn_bbox_loss_, grad_scale=1.0*lr_mult / (config.TRAIN.RPN_BATCH_SIZE)) - ret_group.append(rpn_bbox_loss) - ret_group.append(mx.sym.BlockGrad(bbox_weight)) - - #landmark loss - if landmark: - landmark_diff = rpn_landmark_pred_reshape-landmark_target - landmark_diff = landmark_diff * landmark_weight - rpn_landmark_loss_ = mx.symbol.smooth_l1(name='%s_rpn_landmark_loss_stride%d_'%(prefix,stride), scalar=3.0, data=landmark_diff) - rpn_landmark_loss = mx.sym.MakeLoss(name='%s_rpn_landmark_loss_stride%d'%(prefix,stride), data=rpn_landmark_loss_, grad_scale=0.5*lr_mult / (config.TRAIN.RPN_BATCH_SIZE)) - ret_group.append(rpn_landmark_loss) - ret_group.append(mx.sym.BlockGrad(landmark_weight)) - return ret_group - -def get_ssh_train(): - """ - Region Proposal Network with VGG - :return: Symbol - """ - data = mx.symbol.Variable(name="data") - - # shared convolutional layers - conv_fpn_feat = get_ssh_conv(data) - ret_group = [] - for stride in config.RPN_FEAT_STRIDE: - ret = get_out(conv_fpn_feat, 'face', stride, config.FACE_LANDMARK, lr_mult=1.0) - ret_group += ret - if config.HEAD_BOX: - ret = get_out(conv_fpn_feat, 'head', stride, False, lr_mult=1.0) - ret_group += ret - - return mx.sym.Group(ret_group) - - diff --git a/RetinaFace/rcnn/tools/reeval.py b/RetinaFace/rcnn/tools/reeval.py deleted file mode 100644 index 1bc9c5b..0000000 --- a/RetinaFace/rcnn/tools/reeval.py +++ /dev/null @@ -1,50 +0,0 @@ -import argparse -try: - import cPickle as pickle -except ImportError: - import pickle -import os -import mxnet as mx - -from ..logger import logger -from ..config import config, default, generate_config -from ..dataset import * - - -def reeval(args): - # load imdb - imdb = eval(args.dataset)(args.image_set, args.root_path, args.dataset_path) - - # load detection results - cache_file = os.path.join(imdb.cache_path, imdb.name, 'detections.pkl') - with open(cache_file) as f: - detections = pickle.load(f) - - # eval - imdb.evaluate_detections(detections) - - -def parse_args(): - parser = argparse.ArgumentParser(description='imdb test') - # general - parser.add_argument('--network', help='network name', default=default.network, type=str) - parser.add_argument('--dataset', help='dataset name', default=default.dataset, type=str) - args, rest = parser.parse_known_args() - generate_config(args.network, args.dataset) - parser.add_argument('--image_set', help='image_set name', default=default.image_set, type=str) - parser.add_argument('--root_path', help='output data folder', default=default.root_path, type=str) - parser.add_argument('--dataset_path', help='dataset path', default=default.dataset_path, type=str) - # other - parser.add_argument('--no_shuffle', help='disable random shuffle', action='store_true') - args = parser.parse_args() - return args - - -def main(): - args = parse_args() - logger.info('Called with argument: %s' % args) - reeval(args) - - -if __name__ == '__main__': - main() diff --git a/RetinaFace/rcnn/tools/test_rcnn.py b/RetinaFace/rcnn/tools/test_rcnn.py deleted file mode 100644 index 5e53106..0000000 --- a/RetinaFace/rcnn/tools/test_rcnn.py +++ /dev/null @@ -1,109 +0,0 @@ -import argparse -import pprint -import mxnet as mx - -from ..logger import logger -from ..config import config, default, generate_config -from ..symbol import * -from ..dataset import * -from ..core.loader import TestLoader -from ..core.tester import Predictor, pred_eval -from ..utils.load_model import load_param - - -def test_rcnn(network, dataset, image_set, root_path, dataset_path, - ctx, prefix, epoch, - vis, shuffle, has_rpn, proposal, thresh): - # set config - if has_rpn: - config.TEST.HAS_RPN = True - - # print config - logger.info(pprint.pformat(config)) - - # load symbol and testing data - if has_rpn: - sym = eval('get_' + network + '_test')(num_classes=config.NUM_CLASSES, num_anchors=config.NUM_ANCHORS) - imdb = eval(dataset)(image_set, root_path, dataset_path) - roidb = imdb.gt_roidb() - else: - sym = eval('get_' + network + '_rcnn_test')(num_classes=config.NUM_CLASSES) - imdb = eval(dataset)(image_set, root_path, dataset_path) - gt_roidb = imdb.gt_roidb() - roidb = eval('imdb.' + proposal + '_roidb')(gt_roidb) - - # get test data iter - test_data = TestLoader(roidb, batch_size=1, shuffle=shuffle, has_rpn=has_rpn) - - # load model - arg_params, aux_params = load_param(prefix, epoch, convert=True, ctx=ctx, process=True) - - # infer shape - data_shape_dict = dict(test_data.provide_data) - arg_shape, _, aux_shape = sym.infer_shape(**data_shape_dict) - arg_shape_dict = dict(zip(sym.list_arguments(), arg_shape)) - aux_shape_dict = dict(zip(sym.list_auxiliary_states(), aux_shape)) - - # check parameters - for k in sym.list_arguments(): - if k in data_shape_dict or 'label' in k: - continue - assert k in arg_params, k + ' not initialized' - assert arg_params[k].shape == arg_shape_dict[k], \ - 'shape inconsistent for ' + k + ' inferred ' + str(arg_shape_dict[k]) + ' provided ' + str(arg_params[k].shape) - for k in sym.list_auxiliary_states(): - assert k in aux_params, k + ' not initialized' - assert aux_params[k].shape == aux_shape_dict[k], \ - 'shape inconsistent for ' + k + ' inferred ' + str(aux_shape_dict[k]) + ' provided ' + str(aux_params[k].shape) - - # decide maximum shape - data_names = [k[0] for k in test_data.provide_data] - label_names = None - max_data_shape = [('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))] - if not has_rpn: - max_data_shape.append(('rois', (1, config.TEST.PROPOSAL_POST_NMS_TOP_N + 30, 5))) - - # create predictor - predictor = Predictor(sym, data_names, label_names, - context=ctx, max_data_shapes=max_data_shape, - provide_data=test_data.provide_data, provide_label=test_data.provide_label, - arg_params=arg_params, aux_params=aux_params) - - # start detection - pred_eval(predictor, test_data, imdb, vis=vis, thresh=thresh) - - -def parse_args(): - parser = argparse.ArgumentParser(description='Test a Fast R-CNN network') - # general - parser.add_argument('--network', help='network name', default=default.network, type=str) - parser.add_argument('--dataset', help='dataset name', default=default.dataset, type=str) - args, rest = parser.parse_known_args() - generate_config(args.network, args.dataset) - parser.add_argument('--image_set', help='image_set name', default=default.test_image_set, type=str) - parser.add_argument('--root_path', help='output data folder', default=default.root_path, type=str) - parser.add_argument('--dataset_path', help='dataset path', default=default.dataset_path, type=str) - # testing - parser.add_argument('--prefix', help='model to test with', default=default.rcnn_prefix, type=str) - parser.add_argument('--epoch', help='model to test with', default=default.rcnn_epoch, type=int) - parser.add_argument('--gpu', help='GPU device to test with', default=0, type=int) - # rcnn - parser.add_argument('--vis', help='turn on visualization', action='store_true') - parser.add_argument('--thresh', help='valid detection threshold', default=1e-3, type=float) - parser.add_argument('--shuffle', help='shuffle data on visualization', action='store_true') - parser.add_argument('--has_rpn', help='generate proposals on the fly', action='store_true') - parser.add_argument('--proposal', help='can be ss for selective search or rpn', default='rpn', type=str) - args = parser.parse_args() - return args - - -def main(): - args = parse_args() - logger.info('Called with argument: %s' % args) - ctx = mx.gpu(args.gpu) - test_rcnn(args.network, args.dataset, args.image_set, args.root_path, args.dataset_path, - ctx, args.prefix, args.epoch, - args.vis, args.shuffle, args.has_rpn, args.proposal, args.thresh) - -if __name__ == '__main__': - main() diff --git a/RetinaFace/rcnn/tools/test_rpn.py b/RetinaFace/rcnn/tools/test_rpn.py deleted file mode 100644 index 5622227..0000000 --- a/RetinaFace/rcnn/tools/test_rpn.py +++ /dev/null @@ -1,102 +0,0 @@ -import argparse -import pprint -import mxnet as mx - -from ..logger import logger -from ..config import config, default, generate_config -from ..symbol import * -from ..dataset import * -from ..core.loader import TestLoader -from ..core.tester import Predictor, generate_proposals, test_proposals -from ..utils.load_model import load_param - - -def test_rpn(network, dataset, image_set, root_path, dataset_path, - ctx, prefix, epoch, - vis, shuffle, thresh, test_output=False): - # rpn generate proposal config - config.TEST.HAS_RPN = True - - # print config - logger.info(pprint.pformat(config)) - - # load symbol - sym = eval('get_' + network + '_rpn_test')() - - # load dataset and prepare imdb for training - imdb = eval(dataset)(image_set, root_path, dataset_path) - roidb = imdb.gt_roidb() - test_data = TestLoader(roidb, batch_size=1, shuffle=shuffle, has_rpn=True, withlabel=True) - - # load model - arg_params, aux_params = load_param(prefix, epoch, convert=True, ctx=ctx) - - # infer shape - data_shape_dict = dict(test_data.provide_data) - arg_shape, _, aux_shape = sym.infer_shape(**data_shape_dict) - arg_shape_dict = dict(zip(sym.list_arguments(), arg_shape)) - aux_shape_dict = dict(zip(sym.list_auxiliary_states(), aux_shape)) - - # check parameters - for k in sym.list_arguments(): - if k in data_shape_dict or 'label' in k: - continue - assert k in arg_params, k + ' not initialized' - assert arg_params[k].shape == arg_shape_dict[k], \ - 'shape inconsistent for ' + k + ' inferred ' + str(arg_shape_dict[k]) + ' provided ' + str(arg_params[k].shape) - for k in sym.list_auxiliary_states(): - assert k in aux_params, k + ' not initialized' - assert aux_params[k].shape == aux_shape_dict[k], \ - 'shape inconsistent for ' + k + ' inferred ' + str(aux_shape_dict[k]) + ' provided ' + str(aux_params[k].shape) - - # decide maximum shape - data_names = [k[0] for k in test_data.provide_data] - label_names = None if test_data.provide_label is None else [k[0] for k in test_data.provide_label] - max_data_shape = [('data', (1, 3, max([v[1] for v in config.SCALES]), max([v[1] for v in config.SCALES])))] - - # create predictor - predictor = Predictor(sym, data_names, label_names, - context=ctx, max_data_shapes=max_data_shape, - provide_data=test_data.provide_data, provide_label=test_data.provide_label, - arg_params=arg_params, aux_params=aux_params) - - # start testing - if not test_output: - imdb_boxes = generate_proposals(predictor, test_data, imdb, vis=vis, thresh=thresh) - imdb.evaluate_recall(roidb, candidate_boxes=imdb_boxes) - else: - test_proposals(predictor, test_data, imdb, roidb, vis=vis) - - -def parse_args(): - parser = argparse.ArgumentParser(description='Test a Region Proposal Network') - # general - parser.add_argument('--network', help='network name', default=default.network, type=str) - parser.add_argument('--dataset', help='dataset name', default=default.dataset, type=str) - args, rest = parser.parse_known_args() - generate_config(args.network, args.dataset) - parser.add_argument('--image_set', help='image_set name', default=default.test_image_set, type=str) - parser.add_argument('--root_path', help='output data folder', default=default.root_path, type=str) - parser.add_argument('--dataset_path', help='dataset path', default=default.dataset_path, type=str) - # testing - parser.add_argument('--prefix', help='model to test with', default=default.rpn_prefix, type=str) - parser.add_argument('--epoch', help='model to test with', default=default.rpn_epoch, type=int) - # rpn - parser.add_argument('--gpu', help='GPU device to test with', default=0, type=int) - parser.add_argument('--vis', help='turn on visualization', action='store_true') - parser.add_argument('--thresh', help='rpn proposal threshold', default=0, type=float) - parser.add_argument('--shuffle', help='shuffle data on visualization', action='store_true') - args = parser.parse_args() - return args - - -def main(): - args = parse_args() - logger.info('Called with argument: %s' % args) - ctx = mx.gpu(args.gpu) - test_rpn(args.network, args.dataset, args.image_set, args.root_path, args.dataset_path, - ctx, args.prefix, args.epoch, - args.vis, args.shuffle, args.thresh) - -if __name__ == '__main__': - main() diff --git a/RetinaFace/rcnn/tools/train_rcnn.py b/RetinaFace/rcnn/tools/train_rcnn.py deleted file mode 100644 index 4d2f797..0000000 --- a/RetinaFace/rcnn/tools/train_rcnn.py +++ /dev/null @@ -1,172 +0,0 @@ -import argparse -import pprint -import mxnet as mx - -from ..logger import logger -from ..config import config, default, generate_config -from ..symbol import * -from ..core import callback, metric -from ..core.loader import ROIIter -from ..core.module import MutableModule -from ..processing.bbox_regression import add_bbox_regression_targets -from ..utils.load_data import load_proposal_roidb, merge_roidb, filter_roidb -from ..utils.load_model import load_param - - -def train_rcnn(network, dataset, image_set, root_path, dataset_path, - frequent, kvstore, work_load_list, no_flip, no_shuffle, resume, - ctx, pretrained, epoch, prefix, begin_epoch, end_epoch, - train_shared, lr, lr_step, proposal): - # set up config - config.TRAIN.BATCH_IMAGES = 2 - config.TRAIN.BATCH_ROIS = 128 - if proposal == 'ss': - config.TRAIN.BG_THRESH_LO = 0.1 # reproduce Fast R-CNN - - # load symbol - sym = eval('get_' + network + '_rcnn')(num_classes=config.NUM_CLASSES) - - # setup multi-gpu - batch_size = len(ctx) - input_batch_size = config.TRAIN.BATCH_IMAGES * batch_size - - # print config - logger.info(pprint.pformat(config)) - - # load dataset and prepare imdb for training - image_sets = [iset for iset in image_set.split('+')] - roidbs = [load_proposal_roidb(dataset, image_set, root_path, dataset_path, - proposal=proposal, append_gt=True, flip=not no_flip) - for image_set in image_sets] - roidb = merge_roidb(roidbs) - roidb = filter_roidb(roidb) - means, stds = add_bbox_regression_targets(roidb) - - # load training data - train_data = ROIIter(roidb, batch_size=input_batch_size, shuffle=not no_shuffle, - ctx=ctx, work_load_list=work_load_list, aspect_grouping=config.TRAIN.ASPECT_GROUPING) - - # infer max shape - max_data_shape = [('data', (input_batch_size, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))] - logger.info('providing maximum shape %s' % max_data_shape) - - # infer shape - data_shape_dict = dict(train_data.provide_data + train_data.provide_label) - arg_shape, out_shape, aux_shape = sym.infer_shape(**data_shape_dict) - arg_shape_dict = dict(zip(sym.list_arguments(), arg_shape)) - out_shape_dict = dict(zip(sym.list_outputs(), out_shape)) - aux_shape_dict = dict(zip(sym.list_auxiliary_states(), aux_shape)) - logger.info('output shape %s' % pprint.pformat(out_shape_dict)) - - # load and initialize params - if resume: - arg_params, aux_params = load_param(prefix, begin_epoch, convert=True) - else: - arg_params, aux_params = load_param(pretrained, epoch, convert=True) - arg_params['cls_score_weight'] = mx.random.normal(0, 0.01, shape=arg_shape_dict['cls_score_weight']) - arg_params['cls_score_bias'] = mx.nd.zeros(shape=arg_shape_dict['cls_score_bias']) - arg_params['bbox_pred_weight'] = mx.random.normal(0, 0.001, shape=arg_shape_dict['bbox_pred_weight']) - arg_params['bbox_pred_bias'] = mx.nd.zeros(shape=arg_shape_dict['bbox_pred_bias']) - - # check parameter shapes - for k in sym.list_arguments(): - if k in data_shape_dict: - continue - assert k in arg_params, k + ' not initialized' - assert arg_params[k].shape == arg_shape_dict[k], \ - 'shape inconsistent for ' + k + ' inferred ' + str(arg_shape_dict[k]) + ' provided ' + str(arg_params[k].shape) - for k in sym.list_auxiliary_states(): - assert k in aux_params, k + ' not initialized' - assert aux_params[k].shape == aux_shape_dict[k], \ - 'shape inconsistent for ' + k + ' inferred ' + str(aux_shape_dict[k]) + ' provided ' + str(aux_params[k].shape) - - # prepare training - # create solver - data_names = [k[0] for k in train_data.provide_data] - label_names = [k[0] for k in train_data.provide_label] - if train_shared: - fixed_param_prefix = config.FIXED_PARAMS_SHARED - else: - fixed_param_prefix = config.FIXED_PARAMS - mod = MutableModule(sym, data_names=data_names, label_names=label_names, - logger=logger, context=ctx, work_load_list=work_load_list, - max_data_shapes=max_data_shape, fixed_param_prefix=fixed_param_prefix) - - # decide training params - # metric - eval_metric = metric.RCNNAccMetric() - cls_metric = metric.RCNNLogLossMetric() - bbox_metric = metric.RCNNL1LossMetric() - eval_metrics = mx.metric.CompositeEvalMetric() - for child_metric in [eval_metric, cls_metric, bbox_metric]: - eval_metrics.add(child_metric) - # callback - batch_end_callback = mx.callback.Speedometer(train_data.batch_size, frequent=frequent, auto_reset=False) - epoch_end_callback = callback.do_checkpoint(prefix, means, stds) - # decide learning rate - base_lr = lr - lr_factor = 0.1 - lr_epoch = [int(epoch) for epoch in lr_step.split(',')] - lr_epoch_diff = [epoch - begin_epoch for epoch in lr_epoch if epoch > begin_epoch] - lr = base_lr * (lr_factor ** (len(lr_epoch) - len(lr_epoch_diff))) - lr_iters = [int(epoch * len(roidb) / batch_size) for epoch in lr_epoch_diff] - logger.info('lr %f lr_epoch_diff %s lr_iters %s' % (lr, lr_epoch_diff, lr_iters)) - lr_scheduler = mx.lr_scheduler.MultiFactorScheduler(lr_iters, lr_factor) - # optimizer - optimizer_params = {'momentum': 0.9, - 'wd': 0.0005, - 'learning_rate': lr, - 'lr_scheduler': lr_scheduler, - 'rescale_grad': (1.0 / batch_size), - 'clip_gradient': 5} - - # train - mod.fit(train_data, eval_metric=eval_metrics, epoch_end_callback=epoch_end_callback, - batch_end_callback=batch_end_callback, kvstore=kvstore, - optimizer='sgd', optimizer_params=optimizer_params, - arg_params=arg_params, aux_params=aux_params, begin_epoch=begin_epoch, num_epoch=end_epoch) - - -def parse_args(): - parser = argparse.ArgumentParser(description='Train a Fast R-CNN Network') - # general - parser.add_argument('--network', help='network name', default=default.network, type=str) - parser.add_argument('--dataset', help='dataset name', default=default.dataset, type=str) - args, rest = parser.parse_known_args() - generate_config(args.network, args.dataset) - parser.add_argument('--image_set', help='image_set name', default=default.image_set, type=str) - parser.add_argument('--root_path', help='output data folder', default=default.root_path, type=str) - parser.add_argument('--dataset_path', help='dataset path', default=default.dataset_path, type=str) - # training - parser.add_argument('--frequent', help='frequency of logging', default=default.frequent, type=int) - parser.add_argument('--kvstore', help='the kv-store type', default=default.kvstore, type=str) - parser.add_argument('--work_load_list', help='work load for different devices', default=None, type=list) - parser.add_argument('--no_flip', help='disable flip images', action='store_true') - parser.add_argument('--no_shuffle', help='disable random shuffle', action='store_true') - parser.add_argument('--resume', help='continue training', action='store_true') - # rcnn - parser.add_argument('--gpus', help='GPU device to train with', default='0', type=str) - parser.add_argument('--pretrained', help='pretrained model prefix', default=default.pretrained, type=str) - parser.add_argument('--pretrained_epoch', help='pretrained model epoch', default=default.pretrained_epoch, type=int) - parser.add_argument('--prefix', help='new model prefix', default=default.rcnn_prefix, type=str) - parser.add_argument('--begin_epoch', help='begin epoch of training', default=0, type=int) - parser.add_argument('--end_epoch', help='end epoch of training', default=default.rcnn_epoch, type=int) - parser.add_argument('--lr', help='base learning rate', default=default.rcnn_lr, type=float) - parser.add_argument('--lr_step', help='learning rate steps (in epoch)', default=default.rcnn_lr_step, type=str) - parser.add_argument('--train_shared', help='second round train shared params', action='store_true') - parser.add_argument('--proposal', help='can be ss for selective search or rpn', default='rpn', type=str) - args = parser.parse_args() - return args - - -def main(): - args = parse_args() - logger.info('Called with argument: %s' % args) - ctx = [mx.gpu(int(i)) for i in args.gpus.split(',')] - train_rcnn(args.network, args.dataset, args.image_set, args.root_path, args.dataset_path, - args.frequent, args.kvstore, args.work_load_list, args.no_flip, args.no_shuffle, args.resume, - ctx, args.pretrained, args.pretrained_epoch, args.prefix, args.begin_epoch, args.end_epoch, - train_shared=args.train_shared, lr=args.lr, lr_step=args.lr_step, proposal=args.proposal) - -if __name__ == '__main__': - main() diff --git a/RetinaFace/rcnn/tools/train_rpn.py b/RetinaFace/rcnn/tools/train_rpn.py deleted file mode 100644 index 5a9e296..0000000 --- a/RetinaFace/rcnn/tools/train_rpn.py +++ /dev/null @@ -1,195 +0,0 @@ -import argparse -import logging -import pprint -import mxnet as mx - -from ..config import config, default, generate_config -from ..symbol import * -from ..core import callback, metric -from ..core.loader import AnchorLoaderFPN -from ..core.module import MutableModule -from ..utils.load_data import load_gt_roidb, merge_roidb, filter_roidb -from ..utils.load_model import load_param - - -def train_rpn(network, dataset, image_set, root_path, dataset_path, - frequent, kvstore, work_load_list, no_flip, no_shuffle, resume, - ctx, pretrained, epoch, prefix, begin_epoch, end_epoch, - train_shared, lr, lr_step): - # set up logger - logging.basicConfig() - logger = logging.getLogger() - logger.setLevel(logging.INFO) - - # setup config - assert config.TRAIN.BATCH_IMAGES==1 - - # load symbol - sym = eval('get_' + network + '_rpn')() - feat_sym = [] - for stride in config.RPN_FEAT_STRIDE: - feat_sym.append(sym.get_internals()['rpn_cls_score_stride%s_output' % stride]) - - - # setup multi-gpu - batch_size = len(ctx) - input_batch_size = config.TRAIN.BATCH_IMAGES * batch_size - - # print config - pprint.pprint(config) - - # load dataset and prepare imdb for training - image_sets = [iset for iset in image_set.split('+')] - roidbs = [load_gt_roidb(dataset, image_set, root_path, dataset_path, - flip=not no_flip) - for image_set in image_sets] - roidb = merge_roidb(roidbs) - roidb = filter_roidb(roidb) - - # load training data - #train_data = AnchorLoaderFPN(feat_sym, roidb, batch_size=input_batch_size, shuffle=not no_shuffle, - # ctx=ctx, work_load_list=work_load_list, - # feat_stride=config.RPN_FEAT_STRIDE, anchor_scales=config.ANCHOR_SCALES, - # anchor_ratios=config.ANCHOR_RATIOS, aspect_grouping=config.TRAIN.ASPECT_GROUPING, - # allowed_border=9999) - train_data = AnchorLoaderFPN(feat_sym, roidb, batch_size=input_batch_size, shuffle=not no_shuffle, - ctx=ctx, work_load_list=work_load_list) - - # infer max shape - max_data_shape = [('data', (input_batch_size, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))] - max_data_shape, max_label_shape = train_data.infer_shape(max_data_shape) - print 'providing maximum shape', max_data_shape, max_label_shape - - # infer shape - data_shape_dict = dict(train_data.provide_data + train_data.provide_label) - arg_shape, out_shape, aux_shape = sym.infer_shape(**data_shape_dict) - arg_shape_dict = dict(zip(sym.list_arguments(), arg_shape)) - out_shape_dict = zip(sym.list_outputs(), out_shape) - aux_shape_dict = dict(zip(sym.list_auxiliary_states(), aux_shape)) - print 'output shape' - pprint.pprint(out_shape_dict) - - # load and initialize params - if resume: - arg_params, aux_params = load_param(prefix, begin_epoch, convert=True) - else: - arg_params, aux_params = load_param(pretrained, epoch, convert=True) - init = mx.init.Xavier(factor_type="in", rnd_type='gaussian', magnitude=2) - init_internal = mx.init.Normal(sigma=0.01) - for k in sym.list_arguments(): - if k in data_shape_dict: - continue - if k not in arg_params: - print 'init', k - arg_params[k] = mx.nd.zeros(shape=arg_shape_dict[k]) - if not k.endswith('bias'): - init_internal(k, arg_params[k]) - - for k in sym.list_auxiliary_states(): - if k not in aux_params: - print 'init', k - aux_params[k] = mx.nd.zeros(shape=aux_shape_dict[k]) - init(k, aux_params[k]) - - # check parameter shapes - for k in sym.list_arguments(): - if k in data_shape_dict: - continue - assert k in arg_params, k + ' not initialized' - assert arg_params[k].shape == arg_shape_dict[k], \ - 'shape inconsistent for ' + k + ' inferred ' + str(arg_shape_dict[k]) + ' provided ' + str(arg_params[k].shape) - for k in sym.list_auxiliary_states(): - assert k in aux_params, k + ' not initialized' - assert aux_params[k].shape == aux_shape_dict[k], \ - 'shape inconsistent for ' + k + ' inferred ' + str(aux_shape_dict[k]) + ' provided ' + str(aux_params[k].shape) - - # create solver - data_names = [k[0] for k in train_data.provide_data] - label_names = [k[0] for k in train_data.provide_label] - if train_shared: - fixed_param_prefix = config.FIXED_PARAMS_SHARED - else: - fixed_param_prefix = config.FIXED_PARAMS - mod = MutableModule(sym, data_names=data_names, label_names=label_names, - logger=logger, context=ctx, work_load_list=work_load_list, - max_data_shapes=max_data_shape, max_label_shapes=max_label_shape, - fixed_param_prefix=fixed_param_prefix) - - # decide training params - # metric - eval_metric = metric.RPNAccMetric() - cls_metric = metric.RPNLogLossMetric() - bbox_metric = metric.RPNL1LossMetric() - eval_metrics = mx.metric.CompositeEvalMetric() - for child_metric in [eval_metric,cls_metric,bbox_metric]: - eval_metrics.add(child_metric) - # callback - batch_end_callback = [] - batch_end_callback.append(mx.callback.Speedometer(train_data.batch_size, frequent=frequent)) - epoch_end_callback = mx.callback.do_checkpoint(prefix) - # decide learning rate - base_lr = lr - lr_factor = 0.1 - lr_epoch = [int(epoch) for epoch in lr_step.split(',')] - lr_epoch_diff = [epoch - begin_epoch for epoch in lr_epoch if epoch > begin_epoch] - lr = base_lr * (lr_factor ** (len(lr_epoch) - len(lr_epoch_diff))) - lr_iters = [int(epoch * len(roidb) / batch_size) for epoch in lr_epoch_diff] - print 'lr', lr, 'lr_epoch_diff', lr_epoch_diff, 'lr_iters', lr_iters - lr_scheduler = mx.lr_scheduler.MultiFactorScheduler(lr_iters, lr_factor) - # optimizer - optimizer_params = {'momentum': 0.9, - 'wd': 0.0001, - 'learning_rate': lr, - 'lr_scheduler': lr_scheduler, - 'rescale_grad': (1.0 / batch_size), - 'clip_gradient': 5} - - # train - mod.fit(train_data, eval_metric=eval_metrics, epoch_end_callback=epoch_end_callback, - batch_end_callback=batch_end_callback, kvstore=kvstore, - optimizer='sgd', optimizer_params=optimizer_params, - arg_params=arg_params, aux_params=aux_params, begin_epoch=begin_epoch, num_epoch=end_epoch) - - -def parse_args(): - parser = argparse.ArgumentParser(description='Train a Region Proposal Network') - # general - parser.add_argument('--network', help='network name', default=default.network, type=str) - parser.add_argument('--dataset', help='dataset name', default=default.dataset, type=str) - args, rest = parser.parse_known_args() - generate_config(args.network, args.dataset) - parser.add_argument('--image_set', help='image_set name', default=default.image_set, type=str) - parser.add_argument('--root_path', help='output data folder', default=default.root_path, type=str) - parser.add_argument('--dataset_path', help='dataset path', default=default.dataset_path, type=str) - # training - parser.add_argument('--frequent', help='frequency of logging', default=default.frequent, type=int) - parser.add_argument('--kvstore', help='the kv-store type', default=default.kvstore, type=str) - parser.add_argument('--work_load_list', help='work load for different devices', default=None, type=list) - parser.add_argument('--no_flip', help='disable flip images', action='store_true') - parser.add_argument('--no_shuffle', help='disable random shuffle', action='store_true') - parser.add_argument('--resume', help='continue training', action='store_true') - # rpn - parser.add_argument('--gpus', help='GPU device to train with', default='0', type=str) - parser.add_argument('--pretrained', help='pretrained model prefix', default=default.pretrained, type=str) - parser.add_argument('--pretrained_epoch', help='pretrained model epoch', default=default.pretrained_epoch, type=int) - parser.add_argument('--prefix', help='new model prefix', default=default.rpn_prefix, type=str) - parser.add_argument('--begin_epoch', help='begin epoch of training', default=0, type=int) - parser.add_argument('--end_epoch', help='end epoch of training', default=default.rpn_epoch, type=int) - parser.add_argument('--lr', help='base learning rate', default=default.rpn_lr, type=float) - parser.add_argument('--lr_step', help='learning rate steps (in epoch)', default=default.rpn_lr_step, type=str) - parser.add_argument('--train_shared', help='second round train shared params', action='store_true') - args = parser.parse_args() - return args - - -def main(): - args = parse_args() - print 'Called with argument:', args - ctx = [mx.gpu(int(i)) for i in args.gpus.split(',')] - train_rpn(args.network, args.dataset, args.image_set, args.root_path, args.dataset_path, - args.frequent, args.kvstore, args.work_load_list, args.no_flip, args.no_shuffle, args.resume, - ctx, args.pretrained, args.pretrained_epoch, args.prefix, args.begin_epoch, args.end_epoch, - train_shared=args.train_shared, lr=args.lr, lr_step=args.lr_step) - -if __name__ == '__main__': - main() diff --git a/RetinaFace/retinaface.py b/RetinaFace/retinaface.py deleted file mode 100644 index 624452b..0000000 --- a/RetinaFace/retinaface.py +++ /dev/null @@ -1,698 +0,0 @@ -from __future__ import print_function -import sys -import os -import datetime -import time -import numpy as np -import mxnet as mx -from mxnet import ndarray as nd -import cv2 -#from rcnn import config -from rcnn.logger import logger -#from rcnn.processing.bbox_transform import nonlinear_pred, clip_boxes, landmark_pred -from rcnn.processing.bbox_transform import clip_boxes -from rcnn.processing.generate_anchor import generate_anchors_fpn, anchors_plane -from rcnn.processing.nms import gpu_nms_wrapper, cpu_nms_wrapper -from rcnn.processing.bbox_transform import bbox_overlaps - -class RetinaFace: - def __init__(self, prefix, epoch, ctx_id=0, network='net3', nms=0.4, nocrop=False, decay4 = 0.5, vote=False): - self.ctx_id = ctx_id - self.network = network - self.decay4 = decay4 - self.nms_threshold = nms - self.vote = vote - self.nocrop = nocrop - self.debug = False - self.fpn_keys = [] - self.anchor_cfg = None - pixel_means=[0.0, 0.0, 0.0] - pixel_stds=[1.0, 1.0, 1.0] - pixel_scale = 1.0 - self.preprocess = False - _ratio = (1.,) - fmc = 3 - if network=='ssh' or network=='vgg': - pixel_means=[103.939, 116.779, 123.68] - self.preprocess = True - elif network=='net3': - _ratio = (1.,) - elif network=='net3a': - _ratio = (1.,1.5) - elif network=='net6': #like pyramidbox or s3fd - fmc = 6 - elif network=='net5': #retinaface - fmc = 5 - elif network=='net5a': - fmc = 5 - _ratio = (1.,1.5) - elif network=='net4': - fmc = 4 - elif network=='net4a': - fmc = 4 - _ratio = (1.,1.5) - elif network=='x5': - fmc = 5 - pixel_means=[103.52, 116.28, 123.675] - pixel_stds=[57.375, 57.12, 58.395] - elif network=='x3': - fmc = 3 - pixel_means=[103.52, 116.28, 123.675] - pixel_stds=[57.375, 57.12, 58.395] - elif network=='x3a': - fmc = 3 - _ratio = (1.,1.5) - pixel_means=[103.52, 116.28, 123.675] - pixel_stds=[57.375, 57.12, 58.395] - else: - assert False, 'network setting error %s'%network - - if fmc==3: - self._feat_stride_fpn = [32, 16, 8] - self.anchor_cfg = { - '32': {'SCALES': (32,16), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999}, - '16': {'SCALES': (8,4), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999}, - '8': {'SCALES': (2,1), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999}, - } - elif fmc==4: - self._feat_stride_fpn = [32, 16, 8, 4] - self.anchor_cfg = { - '32': {'SCALES': (32,16), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999}, - '16': {'SCALES': (8,4), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999}, - '8': {'SCALES': (2,1), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999}, - '4': {'SCALES': (2,1), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999}, - } - elif fmc==6: - self._feat_stride_fpn = [128, 64, 32, 16, 8, 4] - self.anchor_cfg = { - '128': {'SCALES': (32,), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999}, - '64': {'SCALES': (16,), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999}, - '32': {'SCALES': (8,), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999}, - '16': {'SCALES': (4,), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999}, - '8': {'SCALES': (2,), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999}, - '4': {'SCALES': (1,), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999}, - } - elif fmc==5: - self._feat_stride_fpn = [64, 32, 16, 8, 4] - self.anchor_cfg = {} - _ass = 2.0**(1.0/3) - _basescale = 1.0 - for _stride in [4, 8, 16, 32, 64]: - key = str(_stride) - value = {'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999} - scales = [] - for _ in range(3): - scales.append(_basescale) - _basescale *= _ass - value['SCALES'] = tuple(scales) - self.anchor_cfg[key] = value - - print(self._feat_stride_fpn, self.anchor_cfg) - - for s in self._feat_stride_fpn: - self.fpn_keys.append('stride%s'%s) - - - dense_anchor = False - #self._anchors_fpn = dict(zip(self.fpn_keys, generate_anchors_fpn(base_size=fpn_base_size, scales=self._scales, ratios=self._ratios))) - self._anchors_fpn = dict(zip(self.fpn_keys, generate_anchors_fpn(dense_anchor=dense_anchor, cfg=self.anchor_cfg))) - for k in self._anchors_fpn: - v = self._anchors_fpn[k].astype(np.float32) - self._anchors_fpn[k] = v - - self._num_anchors = dict(zip(self.fpn_keys, [anchors.shape[0] for anchors in self._anchors_fpn.values()])) - #self._bbox_pred = nonlinear_pred - #self._landmark_pred = landmark_pred - sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) - if self.ctx_id>=0: - self.ctx = mx.gpu(self.ctx_id) - self.nms = gpu_nms_wrapper(self.nms_threshold, self.ctx_id) - else: - self.ctx = mx.cpu() - self.nms = cpu_nms_wrapper(self.nms_threshold) - self.pixel_means = np.array(pixel_means, dtype=np.float32) - self.pixel_stds = np.array(pixel_stds, dtype=np.float32) - self.pixel_scale = float(pixel_scale) - print('means', self.pixel_means) - self.use_landmarks = False - if len(sym)//len(self._feat_stride_fpn)>=3: - self.use_landmarks = True - print('use_landmarks', self.use_landmarks) - self.cascade = 0 - if float(len(sym))//len(self._feat_stride_fpn)>3.0: - self.cascade = 1 - print('cascade', self.cascade) - #self.bbox_stds = [0.1, 0.1, 0.2, 0.2] - #self.landmark_std = 0.1 - self.bbox_stds = [1.0, 1.0, 1.0, 1.0] - self.landmark_std = 1.0 - - if self.debug: - c = len(sym)//len(self._feat_stride_fpn) - sym = sym[(c*0):] - self._feat_stride_fpn = [32,16,8] - print('sym size:', len(sym)) - - image_size = (640, 640) - self.model = mx.mod.Module(symbol=sym, context=self.ctx, label_names = None) - self.model.bind(data_shapes=[('data', (1, 3, image_size[0], image_size[1]))], for_training=False) - self.model.set_params(arg_params, aux_params) - - def get_input(self, img): - im = img.astype(np.float32) - im_tensor = np.zeros((1, 3, im.shape[0], im.shape[1])) - for i in range(3): - im_tensor[0, i, :, :] = (im[:, :, 2 - i]/self.pixel_scale - self.pixel_means[2 - i])/self.pixel_stds[2-i] - #if self.debug: - # timeb = datetime.datetime.now() - # diff = timeb - timea - # print('X2 uses', diff.total_seconds(), 'seconds') - data = nd.array(im_tensor) - return data - - def detect(self, img, threshold=0.5, scales=[1.0], do_flip=False): - #print('in_detect', threshold, scales, do_flip, do_nms) - proposals_list = [] - scores_list = [] - landmarks_list = [] - strides_list = [] - timea = datetime.datetime.now() - flips = [0] - if do_flip: - flips = [0, 1] - - imgs = [img] - if isinstance(img, list): - imgs = img - for img in imgs: - for im_scale in scales: - for flip in flips: - if im_scale!=1.0: - im = cv2.resize(img, None, None, fx=im_scale, fy=im_scale, interpolation=cv2.INTER_LINEAR) - else: - im = img.copy() - if flip: - im = im[:,::-1,:] - if self.nocrop: - if im.shape[0]%32==0: - h = im.shape[0] - else: - h = (im.shape[0]//32+1)*32 - if im.shape[1]%32==0: - w = im.shape[1] - else: - w = (im.shape[1]//32+1)*32 - _im = np.zeros( (h, w, 3), dtype=np.float32 ) - _im[0:im.shape[0], 0:im.shape[1], :] = im - im = _im - else: - im = im.astype(np.float32) - if self.debug: - timeb = datetime.datetime.now() - diff = timeb - timea - print('X1 uses', diff.total_seconds(), 'seconds') - #self.model.bind(data_shapes=[('data', (1, 3, image_size[0], image_size[1]))], for_training=False) - #im_info = [im.shape[0], im.shape[1], im_scale] - im_info = [im.shape[0], im.shape[1]] - im_tensor = np.zeros((1, 3, im.shape[0], im.shape[1])) - for i in range(3): - im_tensor[0, i, :, :] = (im[:, :, 2 - i]/self.pixel_scale - self.pixel_means[2 - i])/self.pixel_stds[2-i] - if self.debug: - timeb = datetime.datetime.now() - diff = timeb - timea - print('X2 uses', diff.total_seconds(), 'seconds') - data = nd.array(im_tensor) - db = mx.io.DataBatch(data=(data,), provide_data=[('data', data.shape)]) - if self.debug: - timeb = datetime.datetime.now() - diff = timeb - timea - print('X3 uses', diff.total_seconds(), 'seconds') - self.model.forward(db, is_train=False) - net_out = self.model.get_outputs() - #post_nms_topN = self._rpn_post_nms_top_n - #min_size_dict = self._rpn_min_size_fpn - - sym_idx = 0 - - for _idx,s in enumerate(self._feat_stride_fpn): - #if len(scales)>1 and s==32 and im_scale==scales[-1]: - # continue - _key = 'stride%s'%s - stride = int(s) - is_cascade = False - if self.cascade: - is_cascade = True - #if self.vote and stride==4 and len(scales)>2 and (im_scale==scales[0]): - # continue - #print('getting', im_scale, stride, idx, len(net_out), data.shape, file=sys.stderr) - scores = net_out[sym_idx].asnumpy() - if self.debug: - timeb = datetime.datetime.now() - diff = timeb - timea - print('A uses', diff.total_seconds(), 'seconds') - #print(scores.shape) - #print('scores',stride, scores.shape, file=sys.stderr) - scores = scores[:, self._num_anchors['stride%s'%s]:, :, :] - - bbox_deltas = net_out[sym_idx+1].asnumpy() - - #if DEBUG: - # print 'im_size: ({}, {})'.format(im_info[0], im_info[1]) - # print 'scale: {}'.format(im_info[2]) - - #_height, _width = int(im_info[0] / stride), int(im_info[1] / stride) - height, width = bbox_deltas.shape[2], bbox_deltas.shape[3] - - A = self._num_anchors['stride%s'%s] - K = height * width - anchors_fpn = self._anchors_fpn['stride%s'%s] - anchors = anchors_plane(height, width, stride, anchors_fpn) - #print((height, width), (_height, _width), anchors.shape, bbox_deltas.shape, scores.shape, file=sys.stderr) - anchors = anchors.reshape((K * A, 4)) - #print('num_anchors', self._num_anchors['stride%s'%s], file=sys.stderr) - #print('HW', (height, width), file=sys.stderr) - #print('anchors_fpn', anchors_fpn.shape, file=sys.stderr) - #print('anchors', anchors.shape, file=sys.stderr) - #print('bbox_deltas', bbox_deltas.shape, file=sys.stderr) - #print('scores', scores.shape, file=sys.stderr) - - - #scores = self._clip_pad(scores, (height, width)) - scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1)) - - #print('pre', bbox_deltas.shape, height, width) - #bbox_deltas = self._clip_pad(bbox_deltas, (height, width)) - #print('after', bbox_deltas.shape, height, width) - bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)) - bbox_pred_len = bbox_deltas.shape[3]//A - #print(bbox_deltas.shape) - bbox_deltas = bbox_deltas.reshape((-1, bbox_pred_len)) - bbox_deltas[:, 0::4] = bbox_deltas[:,0::4] * self.bbox_stds[0] - bbox_deltas[:, 1::4] = bbox_deltas[:,1::4] * self.bbox_stds[1] - bbox_deltas[:, 2::4] = bbox_deltas[:,2::4] * self.bbox_stds[2] - bbox_deltas[:, 3::4] = bbox_deltas[:,3::4] * self.bbox_stds[3] - proposals = self.bbox_pred(anchors, bbox_deltas) - - - #print(anchors.shape, bbox_deltas.shape, A, K, file=sys.stderr) - if is_cascade: - cascade_sym_num = 0 - cls_cascade = False - bbox_cascade = False - __idx = [3,4] - if not self.use_landmarks: - __idx = [2,3] - for diff_idx in __idx: - if sym_idx+diff_idx>=len(net_out): - break - body = net_out[sym_idx+diff_idx].asnumpy() - if body.shape[1]//A==2: #cls branch - if cls_cascade or bbox_cascade: - break - else: - cascade_scores = body[:, self._num_anchors['stride%s'%s]:, :, :] - cascade_scores = cascade_scores.transpose((0, 2, 3, 1)).reshape((-1, 1)) - #scores = (scores+cascade_scores)/2.0 - scores = cascade_scores #TODO? - cascade_sym_num+=1 - cls_cascade = True - #print('find cascade cls at stride', stride) - elif body.shape[1]//A==4: #bbox branch - cascade_deltas = body.transpose((0,2,3,1)).reshape( (-1, bbox_pred_len) ) - cascade_deltas[:, 0::4] = cascade_deltas[:,0::4] * self.bbox_stds[0] - cascade_deltas[:, 1::4] = cascade_deltas[:,1::4] * self.bbox_stds[1] - cascade_deltas[:, 2::4] = cascade_deltas[:,2::4] * self.bbox_stds[2] - cascade_deltas[:, 3::4] = cascade_deltas[:,3::4] * self.bbox_stds[3] - proposals = self.bbox_pred(proposals, cascade_deltas) - cascade_sym_num+=1 - bbox_cascade = True - #print('find cascade bbox at stride', stride) - - - proposals = clip_boxes(proposals, im_info[:2]) - - #if self.vote: - # if im_scale>1.0: - # keep = self._filter_boxes2(proposals, 160*im_scale, -1) - # else: - # keep = self._filter_boxes2(proposals, -1, 100*im_scale) - # if stride==4: - # keep = self._filter_boxes2(proposals, 12*im_scale, -1) - # proposals = proposals[keep, :] - # scores = scores[keep] - - #keep = self._filter_boxes(proposals, min_size_dict['stride%s'%s] * im_info[2]) - #proposals = proposals[keep, :] - #scores = scores[keep] - #print('333', proposals.shape) - if stride==4 and self.decay4<1.0: - scores *= self.decay4 - - scores_ravel = scores.ravel() - #print('__shapes', proposals.shape, scores_ravel.shape) - #print('max score', np.max(scores_ravel)) - order = np.where(scores_ravel>=threshold)[0] - #_scores = scores_ravel[order] - #_order = _scores.argsort()[::-1] - #order = order[_order] - proposals = proposals[order, :] - scores = scores[order] - if flip: - oldx1 = proposals[:, 0].copy() - oldx2 = proposals[:, 2].copy() - proposals[:, 0] = im.shape[1] - oldx2 - 1 - proposals[:, 2] = im.shape[1] - oldx1 - 1 - - proposals[:,0:4] /= im_scale - - proposals_list.append(proposals) - scores_list.append(scores) - if self.nms_threshold<0.0: - _strides = np.empty(shape=(scores.shape), dtype=np.float32) - _strides.fill(stride) - strides_list.append(_strides) - - if not self.vote and self.use_landmarks: - landmark_deltas = net_out[sym_idx+2].asnumpy() - #landmark_deltas = self._clip_pad(landmark_deltas, (height, width)) - landmark_pred_len = landmark_deltas.shape[1]//A - landmark_deltas = landmark_deltas.transpose((0, 2, 3, 1)).reshape((-1, 5, landmark_pred_len//5)) - landmark_deltas *= self.landmark_std - #print(landmark_deltas.shape, landmark_deltas) - landmarks = self.landmark_pred(anchors, landmark_deltas) - landmarks = landmarks[order, :] - - if flip: - landmarks[:,:,0] = im.shape[1] - landmarks[:,:,0] - 1 - #for a in range(5): - # oldx1 = landmarks[:, a].copy() - # landmarks[:,a] = im.shape[1] - oldx1 - 1 - order = [1,0,2,4,3] - flandmarks = landmarks.copy() - for idx, a in enumerate(order): - flandmarks[:,idx,:] = landmarks[:,a,:] - #flandmarks[:, idx*2] = landmarks[:,a*2] - #flandmarks[:, idx*2+1] = landmarks[:,a*2+1] - landmarks = flandmarks - landmarks[:,:,0:2] /= im_scale - #landmarks /= im_scale - #landmarks = landmarks.reshape( (-1, landmark_pred_len) ) - landmarks_list.append(landmarks) - #proposals = np.hstack((proposals, landmarks)) - if self.use_landmarks: - sym_idx += 3 - else: - sym_idx += 2 - if is_cascade: - sym_idx += cascade_sym_num - - if self.debug: - timeb = datetime.datetime.now() - diff = timeb - timea - print('B uses', diff.total_seconds(), 'seconds') - proposals = np.vstack(proposals_list) - landmarks = None - if proposals.shape[0]==0: - if self.use_landmarks: - landmarks = np.zeros( (0,5,2) ) - if self.nms_threshold<0.0: - return np.zeros( (0,6) ), landmarks - else: - return np.zeros( (0,5) ), landmarks - scores = np.vstack(scores_list) - #print('shapes', proposals.shape, scores.shape) - scores_ravel = scores.ravel() - order = scores_ravel.argsort()[::-1] - #if config.TEST.SCORE_THRESH>0.0: - # _count = np.sum(scores_ravel>config.TEST.SCORE_THRESH) - # order = order[:_count] - proposals = proposals[order, :] - scores = scores[order] - if self.nms_threshold<0.0: - strides = np.vstack(strides_list) - strides = strides[order] - if not self.vote and self.use_landmarks: - landmarks = np.vstack(landmarks_list) - landmarks = landmarks[order].astype(np.float32, copy=False) - - if self.nms_threshold>0.0: - pre_det = np.hstack((proposals[:,0:4], scores)).astype(np.float32, copy=False) - if not self.vote: - keep = self.nms(pre_det) - det = np.hstack( (pre_det, proposals[:,4:]) ) - det = det[keep, :] - if self.use_landmarks: - landmarks = landmarks[keep] - else: - det = np.hstack( (pre_det, proposals[:,4:]) ) - det = self.bbox_vote(det) - elif self.nms_threshold<0.0: - det = np.hstack((proposals[:,0:4], scores, strides)).astype(np.float32, copy=False) - else: - det = np.hstack((proposals[:,0:4], scores)).astype(np.float32, copy=False) - - - if self.debug: - timeb = datetime.datetime.now() - diff = timeb - timea - print('C uses', diff.total_seconds(), 'seconds') - return det, landmarks - - def detect_center(self, img, threshold=0.5, scales=[1.0], do_flip=False): - det, landmarks = self.detect(img, threshold, scales, do_flip) - if det.shape[0]==0: - return None, None - bindex = 0 - if det.shape[0]>1: - img_size = np.asarray(img.shape)[0:2] - bounding_box_size = (det[:,2]-det[:,0])*(det[:,3]-det[:,1]) - img_center = img_size / 2 - offsets = np.vstack([ (det[:,0]+det[:,2])/2-img_center[1], (det[:,1]+det[:,3])/2-img_center[0] ]) - offset_dist_squared = np.sum(np.power(offsets,2.0),0) - bindex = np.argmax(bounding_box_size-offset_dist_squared*2.0) # some extra weight on the centering - bbox = det[bindex,:] - landmark = landmarks[bindex, :, :] - return bbox, landmark - - @staticmethod - def check_large_pose(landmark, bbox): - assert landmark.shape==(5,2) - assert len(bbox)==4 - def get_theta(base, x, y): - vx = x-base - vy = y-base - vx[1] *= -1 - vy[1] *= -1 - tx = np.arctan2(vx[1], vx[0]) - ty = np.arctan2(vy[1], vy[0]) - d = ty-tx - d = np.degrees(d) - #print(vx, tx, vy, ty, d) - #if d<-1.*math.pi: - # d+=2*math.pi - #elif d>math.pi: - # d-=2*math.pi - if d<-180.0: - d+=360. - elif d>180.0: - d-=360.0 - return d - landmark = landmark.astype(np.float32) - - theta1 = get_theta(landmark[0], landmark[3], landmark[2]) - theta2 = get_theta(landmark[1], landmark[2], landmark[4]) - #print(va, vb, theta2) - theta3 = get_theta(landmark[0], landmark[2], landmark[1]) - theta4 = get_theta(landmark[1], landmark[0], landmark[2]) - theta5 = get_theta(landmark[3], landmark[4], landmark[2]) - theta6 = get_theta(landmark[4], landmark[2], landmark[3]) - theta7 = get_theta(landmark[3], landmark[2], landmark[0]) - theta8 = get_theta(landmark[4], landmark[1], landmark[2]) - #print(theta1, theta2, theta3, theta4, theta5, theta6, theta7, theta8) - left_score = 0.0 - right_score = 0.0 - up_score = 0.0 - down_score = 0.0 - if theta1<=0.0: - left_score = 10.0 - elif theta2<=0.0: - right_score = 10.0 - else: - left_score = theta2/theta1 - right_score = theta1/theta2 - if theta3<=10.0 or theta4<=10.0: - up_score = 10.0 - else: - up_score = max(theta1/theta3, theta2/theta4) - if theta5<=10.0 or theta6<=10.0: - down_score = 10.0 - else: - down_score = max(theta7/theta5, theta8/theta6) - mleft = (landmark[0][0]+landmark[3][0])/2 - mright = (landmark[1][0]+landmark[4][0])/2 - box_center = ( (bbox[0]+bbox[2])/2, (bbox[1]+bbox[3])/2 ) - ret = 0 - if left_score>=3.0: - ret = 1 - if ret==0 and left_score>=2.0: - if mright<=box_center[0]: - ret = 1 - if ret==0 and right_score>=3.0: - ret = 2 - if ret==0 and right_score>=2.0: - if mleft>=box_center[0]: - ret = 2 - if ret==0 and up_score>=2.0: - ret = 3 - if ret==0 and down_score>=5.0: - ret = 4 - return ret, left_score, right_score, up_score, down_score - - @staticmethod - def _filter_boxes(boxes, min_size): - """ Remove all boxes with any side smaller than min_size """ - ws = boxes[:, 2] - boxes[:, 0] + 1 - hs = boxes[:, 3] - boxes[:, 1] + 1 - keep = np.where((ws >= min_size) & (hs >= min_size))[0] - return keep - - @staticmethod - def _filter_boxes2(boxes, max_size, min_size): - """ Remove all boxes with any side smaller than min_size """ - ws = boxes[:, 2] - boxes[:, 0] + 1 - hs = boxes[:, 3] - boxes[:, 1] + 1 - if max_size>0: - keep = np.where( np.minimum(ws, hs)0: - keep = np.where( np.maximum(ws, hs)>min_size )[0] - return keep - - @staticmethod - def _clip_pad(tensor, pad_shape): - """ - Clip boxes of the pad area. - :param tensor: [n, c, H, W] - :param pad_shape: [h, w] - :return: [n, c, h, w] - """ - H, W = tensor.shape[2:] - h, w = pad_shape - - if h < H or w < W: - tensor = tensor[:, :, :h, :w].copy() - - return tensor - - @staticmethod - def bbox_pred(boxes, box_deltas): - """ - Transform the set of class-agnostic boxes into class-specific boxes - by applying the predicted offsets (box_deltas) - :param boxes: !important [N 4] - :param box_deltas: [N, 4 * num_classes] - :return: [N 4 * num_classes] - """ - if boxes.shape[0] == 0: - return np.zeros((0, box_deltas.shape[1])) - - boxes = boxes.astype(np.float, copy=False) - widths = boxes[:, 2] - boxes[:, 0] + 1.0 - heights = boxes[:, 3] - boxes[:, 1] + 1.0 - ctr_x = boxes[:, 0] + 0.5 * (widths - 1.0) - ctr_y = boxes[:, 1] + 0.5 * (heights - 1.0) - - dx = box_deltas[:, 0:1] - dy = box_deltas[:, 1:2] - dw = box_deltas[:, 2:3] - dh = box_deltas[:, 3:4] - - pred_ctr_x = dx * widths[:, np.newaxis] + ctr_x[:, np.newaxis] - pred_ctr_y = dy * heights[:, np.newaxis] + ctr_y[:, np.newaxis] - pred_w = np.exp(dw) * widths[:, np.newaxis] - pred_h = np.exp(dh) * heights[:, np.newaxis] - - pred_boxes = np.zeros(box_deltas.shape) - # x1 - pred_boxes[:, 0:1] = pred_ctr_x - 0.5 * (pred_w - 1.0) - # y1 - pred_boxes[:, 1:2] = pred_ctr_y - 0.5 * (pred_h - 1.0) - # x2 - pred_boxes[:, 2:3] = pred_ctr_x + 0.5 * (pred_w - 1.0) - # y2 - pred_boxes[:, 3:4] = pred_ctr_y + 0.5 * (pred_h - 1.0) - - if box_deltas.shape[1]>4: - pred_boxes[:,4:] = box_deltas[:,4:] - - return pred_boxes - - @staticmethod - def landmark_pred(boxes, landmark_deltas): - if boxes.shape[0] == 0: - return np.zeros((0, landmark_deltas.shape[1])) - boxes = boxes.astype(np.float, copy=False) - widths = boxes[:, 2] - boxes[:, 0] + 1.0 - heights = boxes[:, 3] - boxes[:, 1] + 1.0 - ctr_x = boxes[:, 0] + 0.5 * (widths - 1.0) - ctr_y = boxes[:, 1] + 0.5 * (heights - 1.0) - pred = landmark_deltas.copy() - for i in range(5): - pred[:,i,0] = landmark_deltas[:,i,0]*widths + ctr_x - pred[:,i,1] = landmark_deltas[:,i,1]*heights + ctr_y - return pred - #preds = [] - #for i in range(landmark_deltas.shape[1]): - # if i%2==0: - # pred = (landmark_deltas[:,i]*widths + ctr_x) - # else: - # pred = (landmark_deltas[:,i]*heights + ctr_y) - # preds.append(pred) - #preds = np.vstack(preds).transpose() - #return preds - - def bbox_vote(self, det): - #order = det[:, 4].ravel().argsort()[::-1] - #det = det[order, :] - if det.shape[0] == 0: - return np.zeros( (0, 5) ) - #dets = np.array([[10, 10, 20, 20, 0.002]]) - #det = np.empty(shape=[0, 5]) - dets = None - while det.shape[0] > 0: - if dets is not None and dets.shape[0]>=750: - break - # IOU - area = (det[:, 2] - det[:, 0] + 1) * (det[:, 3] - det[:, 1] + 1) - xx1 = np.maximum(det[0, 0], det[:, 0]) - yy1 = np.maximum(det[0, 1], det[:, 1]) - xx2 = np.minimum(det[0, 2], det[:, 2]) - yy2 = np.minimum(det[0, 3], det[:, 3]) - w = np.maximum(0.0, xx2 - xx1 + 1) - h = np.maximum(0.0, yy2 - yy1 + 1) - inter = w * h - o = inter / (area[0] + area[:] - inter) - - # nms - merge_index = np.where(o >= self.nms_threshold)[0] - det_accu = det[merge_index, :] - det = np.delete(det, merge_index, 0) - if merge_index.shape[0] <= 1: - if det.shape[0] == 0: - try: - dets = np.row_stack((dets, det_accu)) - except: - dets = det_accu - continue - det_accu[:, 0:4] = det_accu[:, 0:4] * np.tile(det_accu[:, -1:], (1, 4)) - max_score = np.max(det_accu[:, 4]) - det_accu_sum = np.zeros((1, 5)) - det_accu_sum[:, 0:4] = np.sum(det_accu[:, 0:4], - axis=0) / np.sum(det_accu[:, -1:]) - det_accu_sum[:, 4] = max_score - if dets is None: - dets = det_accu_sum - else: - dets = np.row_stack((dets, det_accu_sum)) - dets = dets[0:750, :] - return dets - diff --git a/RetinaFace/test.py b/RetinaFace/test.py deleted file mode 100644 index 748fd01..0000000 --- a/RetinaFace/test.py +++ /dev/null @@ -1,60 +0,0 @@ -import cv2 -import sys -import numpy as np -import datetime -import os -import glob -from retinaface import RetinaFace - -thresh = 0.8 -scales = [1024, 1980] - -count = 1 - -gpuid = 0 -detector = RetinaFace('./model/R50', 0, gpuid, 'net3') - -img = cv2.imread('t1.jpg') -print(img.shape) -im_shape = img.shape -target_size = scales[0] -max_size = scales[1] -im_size_min = np.min(im_shape[0:2]) -im_size_max = np.max(im_shape[0:2]) -#im_scale = 1.0 -#if im_size_min>target_size or im_size_max>max_size: -im_scale = float(target_size) / float(im_size_min) -# prevent bigger axis from being more than max_size: -if np.round(im_scale * im_size_max) > max_size: - im_scale = float(max_size) / float(im_size_max) - -print('im_scale', im_scale) - -scales = [im_scale] -flip = False - -for c in range(count): - faces, landmarks = detector.detect(img, thresh, scales=scales, do_flip=flip) - print(c, faces.shape, landmarks.shape) - -if faces is not None: - print('find', faces.shape[0], 'faces') - for i in range(faces.shape[0]): - #print('score', faces[i][4]) - box = faces[i].astype(np.int) - #color = (255,0,0) - color = (0,0,255) - cv2.rectangle(img, (box[0], box[1]), (box[2], box[3]), color, 2) - if landmarks is not None: - landmark5 = landmarks[i].astype(np.int) - #print(landmark.shape) - for l in range(landmark5.shape[0]): - color = (0,0,255) - if l==0 or l==3: - color = (0,255,0) - cv2.circle(img, (landmark5[l][0], landmark5[l][1]), 1, color, 2) - - filename = './detector_test.jpg' - print('writing', filename) - cv2.imwrite(filename, img) - diff --git a/RetinaFace/test_widerface.py b/RetinaFace/test_widerface.py deleted file mode 100644 index d0a3eeb..0000000 --- a/RetinaFace/test_widerface.py +++ /dev/null @@ -1,201 +0,0 @@ -from __future__ import print_function - -import argparse -import sys -import os -import time -import numpy as np -import mxnet as mx -from mxnet import ndarray as nd -import cv2 -from rcnn.logger import logger -#from rcnn.config import config, default, generate_config -#from rcnn.tools.test_rcnn import test_rcnn -#from rcnn.tools.test_rpn import test_rpn -from rcnn.processing.bbox_transform import nonlinear_pred, clip_boxes, landmark_pred -from rcnn.processing.generate_anchor import generate_anchors_fpn, anchors_plane -from rcnn.processing.nms import gpu_nms_wrapper -from rcnn.processing.bbox_transform import bbox_overlaps -from rcnn.dataset import retinaface -from retinaface import RetinaFace - - -def parse_args(): - parser = argparse.ArgumentParser(description='Test widerface by retinaface detector') - # general - parser.add_argument('--network', help='network name', default='net3', type=str) - parser.add_argument('--dataset', help='dataset name', default='retinaface', type=str) - parser.add_argument('--image-set', help='image_set name', default='val', type=str) - parser.add_argument('--root-path', help='output data folder', default='./data', type=str) - parser.add_argument('--dataset-path', help='dataset path', default='./data/retinaface', type=str) - parser.add_argument('--gpu', help='GPU device to test with', default=0, type=int) - # testing - parser.add_argument('--prefix', help='model to test with', default='', type=str) - parser.add_argument('--epoch', help='model to test with', default=0, type=int) - parser.add_argument('--output', help='output folder', default='./wout', type=str) - parser.add_argument('--nocrop', help='', action='store_true') - parser.add_argument('--thresh', help='valid detection threshold', default=0.02, type=float) - parser.add_argument('--mode', help='test mode, 0 for fast, 1 for accurate', default=1, type=int) - #parser.add_argument('--pyramid', help='enable pyramid test', action='store_true') - #parser.add_argument('--bbox-vote', help='', action='store_true') - parser.add_argument('--part', help='', default=0, type=int) - parser.add_argument('--parts', help='', default=1, type=int) - args = parser.parse_args() - return args - -detector = None -args = None -imgid = -1 - -def get_boxes(roi, pyramid): - global imgid - im = cv2.imread(roi['image']) - do_flip = False - if not pyramid: - target_size = 1200 - max_size = 1600 - #do_flip = True - target_size = 1504 - max_size = 2000 - target_size = 1600 - max_size = 2150 - im_shape = im.shape - im_size_min = np.min(im_shape[0:2]) - im_size_max = np.max(im_shape[0:2]) - im_scale = float(target_size) / float(im_size_min) - # prevent bigger axis from being more than max_size: - if np.round(im_scale * im_size_max) > max_size: - im_scale = float(max_size) / float(im_size_max) - scales = [im_scale] - else: - do_flip = True - #TEST_SCALES = [500, 800, 1200, 1600] - TEST_SCALES = [500, 800, 1100, 1400, 1700] - target_size = 800 - max_size = 1200 - im_shape = im.shape - im_size_min = np.min(im_shape[0:2]) - im_size_max = np.max(im_shape[0:2]) - im_scale = float(target_size) / float(im_size_min) - # prevent bigger axis from being more than max_size: - if np.round(im_scale * im_size_max) > max_size: - im_scale = float(max_size) / float(im_size_max) - scales = [float(scale)/target_size*im_scale for scale in TEST_SCALES] - boxes, landmarks = detector.detect(im, threshold=args.thresh, scales = scales, do_flip=do_flip) - #print(boxes.shape, landmarks.shape) - if imgid>=0 and imgid<100: - font = cv2.FONT_HERSHEY_SIMPLEX - for i in range(boxes.shape[0]): - box = boxes[i] - ibox = box[0:4].copy().astype(np.int) - cv2.rectangle(im, (ibox[0], ibox[1]), (ibox[2], ibox[3]), (255, 0, 0), 2) - #print('box', ibox) - #if len(ibox)>5: - # for l in range(5): - # pp = (ibox[5+l*2], ibox[6+l*2]) - # cv2.circle(im, (pp[0], pp[1]), 1, (0, 0, 255), 1) - blur = box[5] - k = "%.3f"%blur - cv2.putText(im,k,(ibox[0]+2,ibox[1]+14), font, 0.6, (0,255,0), 2) - #landmarks = box[6:21].reshape( (5,3) ) - if landmarks is not None: - for l in range(5): - color = (0,255,0) - landmark = landmarks[i][l] - pp = (int(landmark[0]), int(landmark[1])) - if landmark[2]-0.5<0.0: - color = (0,0,255) - cv2.circle(im, (pp[0], pp[1]), 1, color, 2) - filename = './testimages/%d.jpg'%imgid - cv2.imwrite(filename, im) - print(filename, 'wrote') - imgid+=1 - - return boxes - - -def test(args): - print('test with', args) - global detector - output_folder = args.output - if not os.path.exists(output_folder): - os.mkdir(output_folder) - detector = RetinaFace(args.prefix, args.epoch, args.gpu, network=args.network, nocrop=args.nocrop, vote=args.bbox_vote) - imdb = eval(args.dataset)(args.image_set, args.root_path, args.dataset_path) - roidb = imdb.gt_roidb() - gt_overlaps = np.zeros(0) - overall = [0.0, 0.0] - gt_max = np.array( (0.0, 0.0) ) - num_pos = 0 - print('roidb size', len(roidb)) - - for i in range(len(roidb)): - if i%args.parts!=args.part: - continue - #if i%10==0: - # print('processing', i, file=sys.stderr) - roi = roidb[i] - boxes = get_boxes(roi, args.pyramid) - if 'boxes' in roi: - gt_boxes = roi['boxes'].copy() - gt_areas = (gt_boxes[:, 2] - gt_boxes[:, 0] + 1) * (gt_boxes[:, 3] - gt_boxes[:, 1] + 1) - num_pos += gt_boxes.shape[0] - - overlaps = bbox_overlaps(boxes.astype(np.float), gt_boxes.astype(np.float)) - #print(im_info, gt_boxes.shape, boxes.shape, overlaps.shape, file=sys.stderr) - - _gt_overlaps = np.zeros((gt_boxes.shape[0])) - - if boxes.shape[0]>0: - _gt_overlaps = overlaps.max(axis=0) - #print('max_overlaps', _gt_overlaps, file=sys.stderr) - for j in range(len(_gt_overlaps)): - if _gt_overlaps[j]>0.5: - continue - #print(j, 'failed', gt_boxes[j], 'max_overlap:', _gt_overlaps[j], file=sys.stderr) - - # append recorded IoU coverage level - found = (_gt_overlaps > 0.5).sum() - recall = found / float(gt_boxes.shape[0]) - #print('recall', _recall, gt_boxes.shape[0], boxes.shape[0], gt_areas, 'num:', i, file=sys.stderr) - overall[0]+=found - overall[1]+=gt_boxes.shape[0] - #gt_overlaps = np.hstack((gt_overlaps, _gt_overlaps)) - #_recall = (gt_overlaps >= threshold).sum() / float(num_pos) - recall_all = float(overall[0])/overall[1] - #print('recall_all', _recall, file=sys.stderr) - print('[%d]'%i, 'recall', recall, (gt_boxes.shape[0], boxes.shape[0]), 'all:', recall_all, file=sys.stderr) - else: - print('[%d]'%i, 'detect %d faces'%boxes.shape[0]) - - - _vec = roidb[i]['image'].split('/') - out_dir = os.path.join(output_folder, _vec[-2]) - if not os.path.exists(out_dir): - os.mkdir(out_dir) - out_file = os.path.join(out_dir, _vec[-1].replace('jpg', 'txt')) - with open(out_file, 'w') as f: - name = '/'.join(roidb[i]['image'].split('/')[-2:]) - f.write("%s\n"%(name)) - f.write("%d\n"%(boxes.shape[0])) - for b in range(boxes.shape[0]): - box = boxes[b] - f.write("%d %d %d %d %g \n"%(box[0], box[1], box[2]-box[0], box[3]-box[1], box[4])) - -def main(): - global args - args = parse_args() - args.pyramid = False - args.bbox_vote = False - if args.mode==1: - args.pyramid = True - args.bbox_vote = True - elif args.mode==2: - args.pyramid = True - args.bbox_vote = False - logger.info('Called with argument: %s' % args) - test(args) - -if __name__ == '__main__': - main() - diff --git a/RetinaFace/train.py b/RetinaFace/train.py deleted file mode 100644 index 3d17ffc..0000000 --- a/RetinaFace/train.py +++ /dev/null @@ -1,372 +0,0 @@ -from __future__ import print_function -import sys -import argparse -import os -import pprint -import re -import mxnet as mx -import numpy as np -from mxnet.module import Module -import mxnet.optimizer as optimizer - -from rcnn.logger import logger -from rcnn.config import config, default, generate_config -from rcnn.symbol import * -from rcnn.core import callback, metric -from rcnn.core.loader import CropLoader, CropLoader2 -from rcnn.core.module import MutableModule -from rcnn.utils.load_data import load_gt_roidb, merge_roidb, filter_roidb -from rcnn.utils.load_model import load_param - - -def get_fixed_params(symbol, fixed_param): - if not config.LAYER_FIX: - return [] - fixed_param_names = [] - #for name in symbol.list_arguments(): - # for f in fixed_param: - # if re.match(f, name): - # fixed_param_names.append(name) - #pre = 'mobilenetv20_features_linearbottleneck' - idx = 0 - for name in symbol.list_arguments(): - #print(idx, name) - if idx<7 and name!='data': - fixed_param_names.append(name) - #elif name.startswith('stage1_'): - # fixed_param_names.append(name) - if name.find('upsampling')>=0: - fixed_param_names.append(name) - - idx+=1 - return fixed_param_names - -def train_net(args, ctx, pretrained, epoch, prefix, begin_epoch, end_epoch, - lr=0.001, lr_step='5'): - # setup config - #init_config() - #print(config) - # setup multi-gpu - - input_batch_size = config.TRAIN.BATCH_IMAGES * len(ctx) - - # print config - logger.info(pprint.pformat(config)) - - # load dataset and prepare imdb for training - image_sets = [iset for iset in args.image_set.split('+')] - roidbs = [load_gt_roidb(args.dataset, image_set, args.root_path, args.dataset_path, - flip=not args.no_flip) - for image_set in image_sets] - #roidb = merge_roidb(roidbs) - #roidb = filter_roidb(roidb) - roidb = roidbs[0] - - # load symbol - #sym = eval('get_' + args.network + '_train')(num_classes=config.NUM_CLASSES, num_anchors=config.NUM_ANCHORS) - #feat_sym = sym.get_internals()['rpn_cls_score_output'] - #train_data = AnchorLoader(feat_sym, roidb, batch_size=input_batch_size, shuffle=not args.no_shuffle, - # ctx=ctx, work_load_list=args.work_load_list, - # feat_stride=config.RPN_FEAT_STRIDE, anchor_scales=config.ANCHOR_SCALES, - # anchor_ratios=config.ANCHOR_RATIOS, aspect_grouping=config.TRAIN.ASPECT_GROUPING) - - # load and initialize params - sym = None - if len(pretrained)==0: - arg_params = {} - aux_params = {} - else: - logger.info('loading %s,%d'%(pretrained, epoch)) - sym, arg_params, aux_params = mx.model.load_checkpoint(pretrained, epoch) - #arg_params, aux_params = load_param(pretrained, epoch, convert=True) - #for k in ['rpn_conv_3x3', 'rpn_cls_score', 'rpn_bbox_pred', 'cls_score', 'bbox_pred']: - # _k = k+"_weight" - # if _k in arg_shape_dict: - # v = 0.001 if _k.startswith('bbox_') else 0.01 - # arg_params[_k] = mx.random.normal(0, v, shape=arg_shape_dict[_k]) - # print('init %s with normal %.5f'%(_k,v)) - # _k = k+"_bias" - # if _k in arg_shape_dict: - # arg_params[_k] = mx.nd.zeros(shape=arg_shape_dict[_k]) - # print('init %s with zero'%(_k)) - - sym = eval('get_' + args.network + '_train')(sym) - #print(sym.get_internals()) - feat_sym = [] - for stride in config.RPN_FEAT_STRIDE: - feat_sym.append(sym.get_internals()['face_rpn_cls_score_stride%s_output' % stride]) - - - - train_data = CropLoader(feat_sym, roidb, batch_size=input_batch_size, shuffle=not args.no_shuffle, - ctx=ctx, work_load_list=args.work_load_list) - - - # infer max shape - max_data_shape = [('data', (1, 3, max([v[1] for v in config.SCALES]), max([v[1] for v in config.SCALES])))] - #max_data_shape = [('data', (1, 3, max([v[1] for v in config.SCALES]), max([v[1] for v in config.SCALES])))] - max_data_shape, max_label_shape = train_data.infer_shape(max_data_shape) - max_data_shape.append(('gt_boxes', (1, roidb[0]['max_num_boxes'], 5))) - logger.info('providing maximum shape %s %s' % (max_data_shape, max_label_shape)) - - # infer shape - data_shape_dict = dict(train_data.provide_data + train_data.provide_label) - arg_shape, out_shape, aux_shape = sym.infer_shape(**data_shape_dict) - arg_shape_dict = dict(zip(sym.list_arguments(), arg_shape)) - out_shape_dict = dict(zip(sym.list_outputs(), out_shape)) - aux_shape_dict = dict(zip(sym.list_auxiliary_states(), aux_shape)) - logger.info('output shape %s' % pprint.pformat(out_shape_dict)) - - - for k in arg_shape_dict: - v = arg_shape_dict[k] - if k.find('upsampling')>=0: - print('initializing upsampling_weight', k) - arg_params[k] = mx.nd.zeros(shape=v) - init = mx.init.Initializer() - init._init_bilinear(k, arg_params[k]) - #print(args[k]) - - # check parameter shapes - #for k in sym.list_arguments(): - # if k in data_shape_dict: - # continue - # assert k in arg_params, k + ' not initialized' - # assert arg_params[k].shape == arg_shape_dict[k], \ - # 'shape inconsistent for ' + k + ' inferred ' + str(arg_shape_dict[k]) + ' provided ' + str(arg_params[k].shape) - #for k in sym.list_auxiliary_states(): - # assert k in aux_params, k + ' not initialized' - # assert aux_params[k].shape == aux_shape_dict[k], \ - # 'shape inconsistent for ' + k + ' inferred ' + str(aux_shape_dict[k]) + ' provided ' + str(aux_params[k].shape) - - fixed_param_prefix = config.FIXED_PARAMS - # create solver - data_names = [k[0] for k in train_data.provide_data] - label_names = [k[0] for k in train_data.provide_label] - fixed_param_names = get_fixed_params(sym, fixed_param_prefix) - print('fixed', fixed_param_names, file=sys.stderr) - mod = Module(sym, data_names=data_names, label_names=label_names, - logger=logger, context=ctx, work_load_list=args.work_load_list, - fixed_param_names=fixed_param_names) - - # metric - eval_metrics = mx.metric.CompositeEvalMetric() - mid=0 - for m in range(len(config.RPN_FEAT_STRIDE)): - stride = config.RPN_FEAT_STRIDE[m] - #mid = m*MSTEP - _metric = metric.RPNAccMetric(pred_idx=mid, label_idx=mid+1, name='RPNAcc_s%s'%stride) - eval_metrics.add(_metric) - mid+=2 - #_metric = metric.RPNLogLossMetric(pred_idx=mid, label_idx=mid+1) - #eval_metrics.add(_metric) - - _metric = metric.RPNL1LossMetric(loss_idx=mid, weight_idx=mid+1, name='RPNL1Loss_s%s'%stride) - eval_metrics.add(_metric) - mid+=2 - if config.FACE_LANDMARK: - _metric = metric.RPNL1LossMetric(loss_idx=mid, weight_idx=mid+1, name='RPNLandMarkL1Loss_s%s'%stride) - eval_metrics.add(_metric) - mid+=2 - if config.HEAD_BOX: - _metric = metric.RPNAccMetric(pred_idx=mid, label_idx=mid+1, name='RPNAcc_head_s%s'%stride) - eval_metrics.add(_metric) - mid+=2 - #_metric = metric.RPNLogLossMetric(pred_idx=mid, label_idx=mid+1) - #eval_metrics.add(_metric) - - _metric = metric.RPNL1LossMetric(loss_idx=mid, weight_idx=mid+1, name='RPNL1Loss_head_s%s'%stride) - eval_metrics.add(_metric) - mid+=2 - if config.CASCADE>0: - for _idx in range(config.CASCADE): - if stride in config.CASCADE_CLS_STRIDES: - _metric = metric.RPNAccMetric(pred_idx=mid, label_idx=mid+1, name='RPNAccCAS%d_s%s'%(_idx,stride)) - eval_metrics.add(_metric) - mid+=2 - if stride in config.CASCADE_BBOX_STRIDES: - _metric = metric.RPNL1LossMetric(loss_idx=mid, weight_idx=mid+1, name='RPNL1LossCAS%d_s%s'%(_idx,stride)) - eval_metrics.add(_metric) - mid+=2 - - # callback - #means = np.tile(np.array(config.TRAIN.BBOX_MEANS), config.NUM_CLASSES) - #stds = np.tile(np.array(config.TRAIN.BBOX_STDS), config.NUM_CLASSES) - #epoch_end_callback = callback.do_checkpoint(prefix, means, stds) - epoch_end_callback = None - # decide learning rate - #base_lr = lr - #lr_factor = 0.1 - #lr = base_lr * (lr_factor ** (len(lr_epoch) - len(lr_epoch_diff))) - - lr_epoch = [int(epoch) for epoch in lr_step.split(',')] - lr_epoch_diff = [epoch - begin_epoch for epoch in lr_epoch if epoch > begin_epoch] - lr_iters = [int(epoch * len(roidb) / input_batch_size) for epoch in lr_epoch_diff] - iter_per_epoch = int(len(roidb)/input_batch_size) - - lr_steps = [] - if len(lr_iters)==5: - factors = [0.5, 0.5, 0.4, 0.1, 0.1] - for i in range(5): - lr_steps.append( (lr_iters[i], factors[i]) ) - elif len(lr_iters)==8: #warmup - for li in lr_iters[0:5]: - lr_steps.append( (li, 1.5849) ) - for li in lr_iters[5:]: - lr_steps.append( (li, 0.1) ) - else: - for li in lr_iters: - lr_steps.append( (li, 0.1) ) - #lr_steps = [ (10,0.1), (20, 0.1) ] #XXX - - end_epoch = 10000 - logger.info('lr %f lr_epoch_diff %s lr_steps %s' % (lr, lr_epoch_diff, lr_steps)) - # optimizer - opt = optimizer.SGD(learning_rate=lr, momentum=0.9, wd=args.wd, rescale_grad=1.0/len(ctx), clip_gradient=None) - initializer=mx.init.Xavier() - #initializer = mx.init.Xavier(rnd_type='gaussian', factor_type="out", magnitude=2) #resnet style - - train_data = mx.io.PrefetchingIter(train_data) - - _cb = mx.callback.Speedometer(train_data.batch_size, frequent=args.frequent, auto_reset=False) - global_step = [0] - - def save_model(epoch): - arg, aux = mod.get_params() - all_layers = mod.symbol.get_internals() - outs = [] - for stride in config.RPN_FEAT_STRIDE: - num_anchors = config.RPN_ANCHOR_CFG[str(stride)]['NUM_ANCHORS'] - if config.CASCADE>0: - _name = 'face_rpn_cls_score_stride%d_output' % (stride) - cls_pred = all_layers[_name] - cls_pred = mx.symbol.Reshape(data=cls_pred, shape=(0, 2, -1, 0)) - - cls_pred = mx.symbol.SoftmaxActivation(data=cls_pred, mode="channel") - cls_pred = mx.symbol.Reshape(data=cls_pred, shape=(0, 2 * num_anchors, -1, 0)) - outs.append(cls_pred) - _name = 'face_rpn_bbox_pred_stride%d_output' % stride - rpn_bbox_pred = all_layers[_name] - outs.append(rpn_bbox_pred) - if config.FACE_LANDMARK: - _name = 'face_rpn_landmark_pred_stride%d_output' % stride - rpn_landmark_pred = all_layers[_name] - outs.append(rpn_landmark_pred) - for casid in range(config.CASCADE): - if stride in config.CASCADE_CLS_STRIDES: - _name = 'face_rpn_cls_score_stride%d_cas%d_output' % (stride, casid) - cls_pred = all_layers[_name] - cls_pred = mx.symbol.Reshape(data=cls_pred, shape=(0, 2, -1, 0)) - cls_pred = mx.symbol.SoftmaxActivation(data=cls_pred, mode="channel") - cls_pred = mx.symbol.Reshape(data=cls_pred, shape=(0, 2 * num_anchors, -1, 0)) - outs.append(cls_pred) - if stride in config.CASCADE_BBOX_STRIDES: - _name = 'face_rpn_bbox_pred_stride%d_cas%d_output' % (stride, casid) - bbox_pred = all_layers[_name] - outs.append(bbox_pred) - else: - _name = 'face_rpn_cls_score_stride%d_output' % stride - rpn_cls_score = all_layers[_name] - - - # prepare rpn data - rpn_cls_score_reshape = mx.symbol.Reshape(data=rpn_cls_score, - shape=(0, 2, -1, 0), - name="face_rpn_cls_score_reshape_stride%d" % stride) - - rpn_cls_prob = mx.symbol.SoftmaxActivation(data=rpn_cls_score_reshape, - mode="channel", - name="face_rpn_cls_prob_stride%d" % stride) - rpn_cls_prob_reshape = mx.symbol.Reshape(data=rpn_cls_prob, - shape=(0, 2 * num_anchors, -1, 0), - name='face_rpn_cls_prob_reshape_stride%d' % stride) - _name = 'face_rpn_bbox_pred_stride%d_output' % stride - rpn_bbox_pred = all_layers[_name] - outs.append(rpn_cls_prob_reshape) - outs.append(rpn_bbox_pred) - if config.FACE_LANDMARK: - _name = 'face_rpn_landmark_pred_stride%d_output' % stride - rpn_landmark_pred = all_layers[_name] - outs.append(rpn_landmark_pred) - _sym = mx.sym.Group(outs) - mx.model.save_checkpoint(prefix, epoch, _sym, arg, aux) - - def _batch_callback(param): - #global global_step - _cb(param) - global_step[0]+=1 - mbatch = global_step[0] - for step in lr_steps: - if mbatch==step[0]: - opt.lr *= step[1] - print('lr change to', opt.lr,' in batch', mbatch, file=sys.stderr) - break - - if mbatch%iter_per_epoch==0: - print('saving checkpoint', mbatch, file=sys.stderr) - save_model(0) - if mbatch==lr_steps[-1][0]: - print('saving final checkpoint', mbatch, file=sys.stderr) - save_model(0) - #arg, aux = mod.get_params() - #mx.model.save_checkpoint(prefix, 99, mod.symbol, arg, aux) - sys.exit(0) - - # train - mod.fit(train_data, eval_metric=eval_metrics, epoch_end_callback=epoch_end_callback, - batch_end_callback=_batch_callback, kvstore=args.kvstore, - optimizer=opt, - initializer = initializer, - allow_missing=True, - arg_params=arg_params, aux_params=aux_params, begin_epoch=begin_epoch, num_epoch=end_epoch) - - -def parse_args(): - parser = argparse.ArgumentParser(description='Train RetinaFace') - # general - parser.add_argument('--network', help='network name', default=default.network, type=str) - parser.add_argument('--dataset', help='dataset name', default=default.dataset, type=str) - args, rest = parser.parse_known_args() - generate_config(args.network, args.dataset) - parser.add_argument('--image_set', help='image_set name', default=default.image_set, type=str) - parser.add_argument('--root_path', help='output data folder', default=default.root_path, type=str) - parser.add_argument('--dataset_path', help='dataset path', default=default.dataset_path, type=str) - # training - parser.add_argument('--frequent', help='frequency of logging', default=default.frequent, type=int) - parser.add_argument('--kvstore', help='the kv-store type', default=default.kvstore, type=str) - parser.add_argument('--work_load_list', help='work load for different devices', default=None, type=list) - parser.add_argument('--no_flip', help='disable flip images', action='store_true') - parser.add_argument('--no_shuffle', help='disable random shuffle', action='store_true') - # e2e - #parser.add_argument('--gpus', help='GPU device to train with', default='0,1,2,3', type=str) - parser.add_argument('--pretrained', help='pretrained model prefix', default=default.pretrained, type=str) - parser.add_argument('--pretrained_epoch', help='pretrained model epoch', default=default.pretrained_epoch, type=int) - parser.add_argument('--prefix', help='new model prefix', default=default.prefix, type=str) - parser.add_argument('--begin_epoch', help='begin epoch of training, use with resume', default=0, type=int) - parser.add_argument('--end_epoch', help='end epoch of training', default=default.end_epoch, type=int) - parser.add_argument('--lr', help='base learning rate', default=default.lr, type=float) - parser.add_argument('--lr_step', help='learning rate steps (in epoch)', default=default.lr_step, type=str) - parser.add_argument('--wd', help='weight decay', default=default.wd, type=float) - args = parser.parse_args() - return args - - -def main(): - args = parse_args() - logger.info('Called with argument: %s' % args) - #ctx = [mx.gpu(int(i)) for i in args.gpus.split(',')] - ctx = [] - cvd = os.environ['CUDA_VISIBLE_DEVICES'].strip() - if len(cvd)>0: - for i in range(len(cvd.split(','))): - ctx.append(mx.gpu(i)) - if len(ctx)==0: - ctx = [mx.cpu()] - print('use cpu') - else: - print('gpu num:', len(ctx)) - train_net(args, ctx, args.pretrained, args.pretrained_epoch, args.prefix, args.begin_epoch, args.end_epoch, - lr=args.lr, lr_step=args.lr_step) - -if __name__ == '__main__': - main() diff --git a/RetinaFaceAntiCov/retinaface_cov.py b/RetinaFaceAntiCov/retinaface_cov.py deleted file mode 100644 index 57d1ef2..0000000 --- a/RetinaFaceAntiCov/retinaface_cov.py +++ /dev/null @@ -1,637 +0,0 @@ -from __future__ import print_function -import sys -import os -import datetime -import time -import numpy as np -import mxnet as mx -from mxnet import ndarray as nd -import cv2 -#from rcnn import config -#from rcnn.processing.bbox_transform import nonlinear_pred, clip_boxes, landmark_pred -from rcnn.processing.bbox_transform import clip_boxes -from rcnn.processing.generate_anchor import generate_anchors_fpn, anchors_plane -from rcnn.processing.nms import gpu_nms_wrapper, cpu_nms_wrapper -from rcnn.processing.bbox_transform import bbox_overlaps - -class RetinaFaceCoV: - def __init__(self, prefix, epoch, ctx_id=0, network='net3', nms=0.4, nocrop=False): - self.ctx_id = ctx_id - self.network = network - self.nms_threshold = nms - self.nocrop = nocrop - self.debug = False - self.fpn_keys = [] - self.anchor_cfg = None - pixel_means=[0.0, 0.0, 0.0] - pixel_stds=[1.0, 1.0, 1.0] - pixel_scale = 1.0 - self.bbox_stds = [1.0, 1.0, 1.0, 1.0] - self.landmark_std = 1.0 - self.preprocess = False - _ratio = (1.,) - fmc = 3 - if network=='ssh' or network=='vgg': - pixel_means=[103.939, 116.779, 123.68] - self.preprocess = True - elif network=='net3': - _ratio = (1.,) - elif network=='net3l': - _ratio = (1.,) - self.landmark_std = 0.2 - elif network=='net3a': - _ratio = (1.,1.5) - elif network=='net6': #like pyramidbox or s3fd - fmc = 6 - elif network=='net5': #retinaface - fmc = 5 - elif network=='net5a': - fmc = 5 - _ratio = (1.,1.5) - elif network=='net4': - fmc = 4 - elif network=='net4a': - fmc = 4 - _ratio = (1.,1.5) - elif network=='x5': - fmc = 5 - pixel_means=[103.52, 116.28, 123.675] - pixel_stds=[57.375, 57.12, 58.395] - elif network=='x3': - fmc = 3 - pixel_means=[103.52, 116.28, 123.675] - pixel_stds=[57.375, 57.12, 58.395] - elif network=='x3a': - fmc = 3 - _ratio = (1.,1.5) - pixel_means=[103.52, 116.28, 123.675] - pixel_stds=[57.375, 57.12, 58.395] - else: - assert False, 'network setting error %s'%network - - if fmc==3: - self._feat_stride_fpn = [32, 16, 8] - self.anchor_cfg = { - '32': {'SCALES': (32,16), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999}, - '16': {'SCALES': (8,4), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999}, - '8': {'SCALES': (2,1), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999}, - } - elif fmc==4: - self._feat_stride_fpn = [32, 16, 8, 4] - self.anchor_cfg = { - '32': {'SCALES': (32,16), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999}, - '16': {'SCALES': (8,4), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999}, - '8': {'SCALES': (2,1), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999}, - '4': {'SCALES': (2,1), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999}, - } - elif fmc==6: - self._feat_stride_fpn = [128, 64, 32, 16, 8, 4] - self.anchor_cfg = { - '128': {'SCALES': (32,), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999}, - '64': {'SCALES': (16,), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999}, - '32': {'SCALES': (8,), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999}, - '16': {'SCALES': (4,), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999}, - '8': {'SCALES': (2,), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999}, - '4': {'SCALES': (1,), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999}, - } - elif fmc==5: - self._feat_stride_fpn = [64, 32, 16, 8, 4] - self.anchor_cfg = {} - _ass = 2.0**(1.0/3) - _basescale = 1.0 - for _stride in [4, 8, 16, 32, 64]: - key = str(_stride) - value = {'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999} - scales = [] - for _ in range(3): - scales.append(_basescale) - _basescale *= _ass - value['SCALES'] = tuple(scales) - self.anchor_cfg[key] = value - - #print(self._feat_stride_fpn, self.anchor_cfg) - - for s in self._feat_stride_fpn: - self.fpn_keys.append('stride%s'%s) - - - dense_anchor = False - #self._anchors_fpn = dict(zip(self.fpn_keys, generate_anchors_fpn(base_size=fpn_base_size, scales=self._scales, ratios=self._ratios))) - self._anchors_fpn = dict(zip(self.fpn_keys, generate_anchors_fpn(dense_anchor=dense_anchor, cfg=self.anchor_cfg))) - for k in self._anchors_fpn: - v = self._anchors_fpn[k].astype(np.float32) - self._anchors_fpn[k] = v - - self._num_anchors = dict(zip(self.fpn_keys, [anchors.shape[0] for anchors in self._anchors_fpn.values()])) - #self._bbox_pred = nonlinear_pred - #self._landmark_pred = landmark_pred - sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) - if self.ctx_id>=0: - self.ctx = mx.gpu(self.ctx_id) - self.nms = gpu_nms_wrapper(self.nms_threshold, self.ctx_id) - else: - self.ctx = mx.cpu() - self.nms = cpu_nms_wrapper(self.nms_threshold) - self.pixel_means = np.array(pixel_means, dtype=np.float32) - self.pixel_stds = np.array(pixel_stds, dtype=np.float32) - self.pixel_scale = float(pixel_scale) - #print('means', self.pixel_means) - self.use_landmarks = True - #print('use_landmarks', self.use_landmarks) - self.cascade = 0 - - if self.debug: - c = len(sym)//len(self._feat_stride_fpn) - sym = sym[(c*0):] - self._feat_stride_fpn = [32,16,8] - #print('sym size:', len(sym)) - - image_size = (640, 640) - self.model = mx.mod.Module(symbol=sym, context=self.ctx, label_names = None) - self.model.bind(data_shapes=[('data', (1, 3, image_size[0], image_size[1]))], for_training=False) - self.model.set_params(arg_params, aux_params) - - def get_input(self, img): - im = img.astype(np.float32) - im_tensor = np.zeros((1, 3, im.shape[0], im.shape[1])) - for i in range(3): - im_tensor[0, i, :, :] = (im[:, :, 2 - i]/self.pixel_scale - self.pixel_means[2 - i])/self.pixel_stds[2-i] - #if self.debug: - # timeb = datetime.datetime.now() - # diff = timeb - timea - # print('X2 uses', diff.total_seconds(), 'seconds') - data = nd.array(im_tensor) - return data - - def detect(self, img, threshold=0.5, scales=[1.0], do_flip=False): - #print('in_detect', threshold, scales, do_flip, do_nms) - proposals_list = [] - scores_list = [] - mask_scores_list = [] - landmarks_list = [] - strides_list = [] - timea = datetime.datetime.now() - flips = [0] - if do_flip: - flips = [0, 1] - - imgs = [img] - if isinstance(img, list): - imgs = img - for img in imgs: - for im_scale in scales: - for flip in flips: - if im_scale!=1.0: - im = cv2.resize(img, None, None, fx=im_scale, fy=im_scale, interpolation=cv2.INTER_LINEAR) - else: - im = img.copy() - if flip: - im = im[:,::-1,:] - if self.nocrop: - if im.shape[0]%32==0: - h = im.shape[0] - else: - h = (im.shape[0]//32+1)*32 - if im.shape[1]%32==0: - w = im.shape[1] - else: - w = (im.shape[1]//32+1)*32 - _im = np.zeros( (h, w, 3), dtype=np.float32 ) - _im[0:im.shape[0], 0:im.shape[1], :] = im - im = _im - else: - im = im.astype(np.float32) - if self.debug: - timeb = datetime.datetime.now() - diff = timeb - timea - print('X1 uses', diff.total_seconds(), 'seconds') - #self.model.bind(data_shapes=[('data', (1, 3, image_size[0], image_size[1]))], for_training=False) - #im_info = [im.shape[0], im.shape[1], im_scale] - im_info = [im.shape[0], im.shape[1]] - im_tensor = np.zeros((1, 3, im.shape[0], im.shape[1])) - for i in range(3): - im_tensor[0, i, :, :] = (im[:, :, 2 - i]/self.pixel_scale - self.pixel_means[2 - i])/self.pixel_stds[2-i] - if self.debug: - timeb = datetime.datetime.now() - diff = timeb - timea - print('X2 uses', diff.total_seconds(), 'seconds') - data = nd.array(im_tensor) - db = mx.io.DataBatch(data=(data,), provide_data=[('data', data.shape)]) - if self.debug: - timeb = datetime.datetime.now() - diff = timeb - timea - print('X3 uses', diff.total_seconds(), 'seconds') - self.model.forward(db, is_train=False) - net_out = self.model.get_outputs() - #post_nms_topN = self._rpn_post_nms_top_n - #min_size_dict = self._rpn_min_size_fpn - - sym_idx = 0 - - for _idx,s in enumerate(self._feat_stride_fpn): - #if len(scales)>1 and s==32 and im_scale==scales[-1]: - # continue - _key = 'stride%s'%s - stride = int(s) - is_cascade = False - #if self.vote and stride==4 and len(scales)>2 and (im_scale==scales[0]): - # continue - #print('getting', im_scale, stride, idx, len(net_out), data.shape, file=sys.stderr) - scores = net_out[sym_idx].asnumpy() - type_scores = net_out[sym_idx+3].asnumpy() - print(scores.shape, type_scores.shape) - if self.debug: - timeb = datetime.datetime.now() - diff = timeb - timea - print('A uses', diff.total_seconds(), 'seconds') - A = self._num_anchors['stride%s'%s] - #print(scores.shape) - #print('scores',stride, scores.shape, file=sys.stderr) - scores = scores[:, A:, :, :] - mask_scores = type_scores[:, A*2:, :, :] #x, A, x, x - - bbox_deltas = net_out[sym_idx+1].asnumpy() - - #if DEBUG: - # print 'im_size: ({}, {})'.format(im_info[0], im_info[1]) - # print 'scale: {}'.format(im_info[2]) - - #_height, _width = int(im_info[0] / stride), int(im_info[1] / stride) - height, width = bbox_deltas.shape[2], bbox_deltas.shape[3] - - K = height * width - anchors_fpn = self._anchors_fpn['stride%s'%s] - anchors = anchors_plane(height, width, stride, anchors_fpn) - #print((height, width), (_height, _width), anchors.shape, bbox_deltas.shape, scores.shape, file=sys.stderr) - anchors = anchors.reshape((K * A, 4)) - #print('num_anchors', self._num_anchors['stride%s'%s], file=sys.stderr) - #print('HW', (height, width), file=sys.stderr) - #print('anchors_fpn', anchors_fpn.shape, file=sys.stderr) - #print('anchors', anchors.shape, file=sys.stderr) - #print('bbox_deltas', bbox_deltas.shape, file=sys.stderr) - #print('scores', scores.shape, file=sys.stderr) - - - #scores = self._clip_pad(scores, (height, width)) - scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1)) - mask_scores = mask_scores.transpose((0, 2, 3, 1)).reshape((-1, 1)) - - #print('pre', bbox_deltas.shape, height, width) - #bbox_deltas = self._clip_pad(bbox_deltas, (height, width)) - #print('after', bbox_deltas.shape, height, width) - bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)) - bbox_pred_len = bbox_deltas.shape[3]//A - #print(bbox_deltas.shape) - bbox_deltas = bbox_deltas.reshape((-1, bbox_pred_len)) - bbox_deltas[:, 0::4] = bbox_deltas[:,0::4] * self.bbox_stds[0] - bbox_deltas[:, 1::4] = bbox_deltas[:,1::4] * self.bbox_stds[1] - bbox_deltas[:, 2::4] = bbox_deltas[:,2::4] * self.bbox_stds[2] - bbox_deltas[:, 3::4] = bbox_deltas[:,3::4] * self.bbox_stds[3] - proposals = self.bbox_pred(anchors, bbox_deltas) - - - - proposals = clip_boxes(proposals, im_info[:2]) - - #if self.vote: - # if im_scale>1.0: - # keep = self._filter_boxes2(proposals, 160*im_scale, -1) - # else: - # keep = self._filter_boxes2(proposals, -1, 100*im_scale) - # if stride==4: - # keep = self._filter_boxes2(proposals, 12*im_scale, -1) - # proposals = proposals[keep, :] - # scores = scores[keep] - - #keep = self._filter_boxes(proposals, min_size_dict['stride%s'%s] * im_info[2]) - #proposals = proposals[keep, :] - #scores = scores[keep] - #print('333', proposals.shape) - if stride==4 and self.decay4<1.0: - scores *= self.decay4 - - scores_ravel = scores.ravel() - #mask_scores_ravel = mask_scores.ravel() - #print('__shapes', proposals.shape, scores_ravel.shape) - #print('max score', np.max(scores_ravel)) - order = np.where(scores_ravel>=threshold)[0] - #_scores = scores_ravel[order] - #_order = _scores.argsort()[::-1] - #order = order[_order] - proposals = proposals[order, :] - scores = scores[order] - mask_scores = mask_scores[order] - if flip: - oldx1 = proposals[:, 0].copy() - oldx2 = proposals[:, 2].copy() - proposals[:, 0] = im.shape[1] - oldx2 - 1 - proposals[:, 2] = im.shape[1] - oldx1 - 1 - - proposals[:,0:4] /= im_scale - - proposals_list.append(proposals) - scores_list.append(scores) - mask_scores_list.append(mask_scores) - - landmark_deltas = net_out[sym_idx+2].asnumpy() - #landmark_deltas = self._clip_pad(landmark_deltas, (height, width)) - landmark_pred_len = landmark_deltas.shape[1]//A - landmark_deltas = landmark_deltas.transpose((0, 2, 3, 1)).reshape((-1, 5, landmark_pred_len//5)) - landmark_deltas *= self.landmark_std - #print(landmark_deltas.shape, landmark_deltas) - landmarks = self.landmark_pred(anchors, landmark_deltas) - landmarks = landmarks[order, :] - - if flip: - landmarks[:,:,0] = im.shape[1] - landmarks[:,:,0] - 1 - #for a in range(5): - # oldx1 = landmarks[:, a].copy() - # landmarks[:,a] = im.shape[1] - oldx1 - 1 - order = [1,0,2,4,3] - flandmarks = landmarks.copy() - for idx, a in enumerate(order): - flandmarks[:,idx,:] = landmarks[:,a,:] - #flandmarks[:, idx*2] = landmarks[:,a*2] - #flandmarks[:, idx*2+1] = landmarks[:,a*2+1] - landmarks = flandmarks - landmarks[:,:,0:2] /= im_scale - #landmarks /= im_scale - #landmarks = landmarks.reshape( (-1, landmark_pred_len) ) - landmarks_list.append(landmarks) - #proposals = np.hstack((proposals, landmarks)) - sym_idx += 4 - - if self.debug: - timeb = datetime.datetime.now() - diff = timeb - timea - print('B uses', diff.total_seconds(), 'seconds') - proposals = np.vstack(proposals_list) - landmarks = None - if proposals.shape[0]==0: - landmarks = np.zeros( (0,5,2) ) - return np.zeros( (0,6) ), landmarks - scores = np.vstack(scores_list) - mask_scores = np.vstack(mask_scores_list) - #print('shapes', proposals.shape, scores.shape) - scores_ravel = scores.ravel() - order = scores_ravel.argsort()[::-1] - #if config.TEST.SCORE_THRESH>0.0: - # _count = np.sum(scores_ravel>config.TEST.SCORE_THRESH) - # order = order[:_count] - proposals = proposals[order, :] - scores = scores[order] - mask_scores = mask_scores[order] - landmarks = np.vstack(landmarks_list) - landmarks = landmarks[order].astype(np.float32, copy=False) - - pre_det = np.hstack((proposals[:,0:4], scores)).astype(np.float32, copy=False) - keep = self.nms(pre_det) - det = np.hstack( (pre_det, mask_scores) ) - det = det[keep, :] - landmarks = landmarks[keep] - - - if self.debug: - timeb = datetime.datetime.now() - diff = timeb - timea - print('C uses', diff.total_seconds(), 'seconds') - return det, landmarks - - def detect_center(self, img, threshold=0.5, scales=[1.0], do_flip=False): - det, landmarks = self.detect(img, threshold, scales, do_flip) - if det.shape[0]==0: - return None, None - bindex = 0 - if det.shape[0]>1: - img_size = np.asarray(img.shape)[0:2] - bounding_box_size = (det[:,2]-det[:,0])*(det[:,3]-det[:,1]) - img_center = img_size / 2 - offsets = np.vstack([ (det[:,0]+det[:,2])/2-img_center[1], (det[:,1]+det[:,3])/2-img_center[0] ]) - offset_dist_squared = np.sum(np.power(offsets,2.0),0) - bindex = np.argmax(bounding_box_size-offset_dist_squared*2.0) # some extra weight on the centering - bbox = det[bindex,:] - landmark = landmarks[bindex, :, :] - return bbox, landmark - - @staticmethod - def check_large_pose(landmark, bbox): - assert landmark.shape==(5,2) - assert len(bbox)==4 - def get_theta(base, x, y): - vx = x-base - vy = y-base - vx[1] *= -1 - vy[1] *= -1 - tx = np.arctan2(vx[1], vx[0]) - ty = np.arctan2(vy[1], vy[0]) - d = ty-tx - d = np.degrees(d) - #print(vx, tx, vy, ty, d) - #if d<-1.*math.pi: - # d+=2*math.pi - #elif d>math.pi: - # d-=2*math.pi - if d<-180.0: - d+=360. - elif d>180.0: - d-=360.0 - return d - landmark = landmark.astype(np.float32) - - theta1 = get_theta(landmark[0], landmark[3], landmark[2]) - theta2 = get_theta(landmark[1], landmark[2], landmark[4]) - #print(va, vb, theta2) - theta3 = get_theta(landmark[0], landmark[2], landmark[1]) - theta4 = get_theta(landmark[1], landmark[0], landmark[2]) - theta5 = get_theta(landmark[3], landmark[4], landmark[2]) - theta6 = get_theta(landmark[4], landmark[2], landmark[3]) - theta7 = get_theta(landmark[3], landmark[2], landmark[0]) - theta8 = get_theta(landmark[4], landmark[1], landmark[2]) - #print(theta1, theta2, theta3, theta4, theta5, theta6, theta7, theta8) - left_score = 0.0 - right_score = 0.0 - up_score = 0.0 - down_score = 0.0 - if theta1<=0.0: - left_score = 10.0 - elif theta2<=0.0: - right_score = 10.0 - else: - left_score = theta2/theta1 - right_score = theta1/theta2 - if theta3<=10.0 or theta4<=10.0: - up_score = 10.0 - else: - up_score = max(theta1/theta3, theta2/theta4) - if theta5<=10.0 or theta6<=10.0: - down_score = 10.0 - else: - down_score = max(theta7/theta5, theta8/theta6) - mleft = (landmark[0][0]+landmark[3][0])/2 - mright = (landmark[1][0]+landmark[4][0])/2 - box_center = ( (bbox[0]+bbox[2])/2, (bbox[1]+bbox[3])/2 ) - ret = 0 - if left_score>=3.0: - ret = 1 - if ret==0 and left_score>=2.0: - if mright<=box_center[0]: - ret = 1 - if ret==0 and right_score>=3.0: - ret = 2 - if ret==0 and right_score>=2.0: - if mleft>=box_center[0]: - ret = 2 - if ret==0 and up_score>=2.0: - ret = 3 - if ret==0 and down_score>=5.0: - ret = 4 - return ret, left_score, right_score, up_score, down_score - - @staticmethod - def _filter_boxes(boxes, min_size): - """ Remove all boxes with any side smaller than min_size """ - ws = boxes[:, 2] - boxes[:, 0] + 1 - hs = boxes[:, 3] - boxes[:, 1] + 1 - keep = np.where((ws >= min_size) & (hs >= min_size))[0] - return keep - - @staticmethod - def _filter_boxes2(boxes, max_size, min_size): - """ Remove all boxes with any side smaller than min_size """ - ws = boxes[:, 2] - boxes[:, 0] + 1 - hs = boxes[:, 3] - boxes[:, 1] + 1 - if max_size>0: - keep = np.where( np.minimum(ws, hs)0: - keep = np.where( np.maximum(ws, hs)>min_size )[0] - return keep - - @staticmethod - def _clip_pad(tensor, pad_shape): - """ - Clip boxes of the pad area. - :param tensor: [n, c, H, W] - :param pad_shape: [h, w] - :return: [n, c, h, w] - """ - H, W = tensor.shape[2:] - h, w = pad_shape - - if h < H or w < W: - tensor = tensor[:, :, :h, :w].copy() - - return tensor - - @staticmethod - def bbox_pred(boxes, box_deltas): - """ - Transform the set of class-agnostic boxes into class-specific boxes - by applying the predicted offsets (box_deltas) - :param boxes: !important [N 4] - :param box_deltas: [N, 4 * num_classes] - :return: [N 4 * num_classes] - """ - if boxes.shape[0] == 0: - return np.zeros((0, box_deltas.shape[1])) - - boxes = boxes.astype(np.float, copy=False) - widths = boxes[:, 2] - boxes[:, 0] + 1.0 - heights = boxes[:, 3] - boxes[:, 1] + 1.0 - ctr_x = boxes[:, 0] + 0.5 * (widths - 1.0) - ctr_y = boxes[:, 1] + 0.5 * (heights - 1.0) - - dx = box_deltas[:, 0:1] - dy = box_deltas[:, 1:2] - dw = box_deltas[:, 2:3] - dh = box_deltas[:, 3:4] - - pred_ctr_x = dx * widths[:, np.newaxis] + ctr_x[:, np.newaxis] - pred_ctr_y = dy * heights[:, np.newaxis] + ctr_y[:, np.newaxis] - pred_w = np.exp(dw) * widths[:, np.newaxis] - pred_h = np.exp(dh) * heights[:, np.newaxis] - - pred_boxes = np.zeros(box_deltas.shape) - # x1 - pred_boxes[:, 0:1] = pred_ctr_x - 0.5 * (pred_w - 1.0) - # y1 - pred_boxes[:, 1:2] = pred_ctr_y - 0.5 * (pred_h - 1.0) - # x2 - pred_boxes[:, 2:3] = pred_ctr_x + 0.5 * (pred_w - 1.0) - # y2 - pred_boxes[:, 3:4] = pred_ctr_y + 0.5 * (pred_h - 1.0) - - if box_deltas.shape[1]>4: - pred_boxes[:,4:] = box_deltas[:,4:] - - return pred_boxes - - @staticmethod - def landmark_pred(boxes, landmark_deltas): - if boxes.shape[0] == 0: - return np.zeros((0, landmark_deltas.shape[1])) - boxes = boxes.astype(np.float, copy=False) - widths = boxes[:, 2] - boxes[:, 0] + 1.0 - heights = boxes[:, 3] - boxes[:, 1] + 1.0 - ctr_x = boxes[:, 0] + 0.5 * (widths - 1.0) - ctr_y = boxes[:, 1] + 0.5 * (heights - 1.0) - pred = landmark_deltas.copy() - for i in range(5): - pred[:,i,0] = landmark_deltas[:,i,0]*widths + ctr_x - pred[:,i,1] = landmark_deltas[:,i,1]*heights + ctr_y - return pred - #preds = [] - #for i in range(landmark_deltas.shape[1]): - # if i%2==0: - # pred = (landmark_deltas[:,i]*widths + ctr_x) - # else: - # pred = (landmark_deltas[:,i]*heights + ctr_y) - # preds.append(pred) - #preds = np.vstack(preds).transpose() - #return preds - - def vote(self, det): - #order = det[:, 4].ravel().argsort()[::-1] - #det = det[order, :] - if det.shape[0] == 0: - return np.zeros( (0, 5) ) - #dets = np.array([[10, 10, 20, 20, 0.002]]) - #det = np.empty(shape=[0, 5]) - dets = None - while det.shape[0] > 0: - if dets is not None and dets.shape[0]>=750: - break - # IOU - area = (det[:, 2] - det[:, 0] + 1) * (det[:, 3] - det[:, 1] + 1) - xx1 = np.maximum(det[0, 0], det[:, 0]) - yy1 = np.maximum(det[0, 1], det[:, 1]) - xx2 = np.minimum(det[0, 2], det[:, 2]) - yy2 = np.minimum(det[0, 3], det[:, 3]) - w = np.maximum(0.0, xx2 - xx1 + 1) - h = np.maximum(0.0, yy2 - yy1 + 1) - inter = w * h - o = inter / (area[0] + area[:] - inter) - - # nms - merge_index = np.where(o >= self.nms_threshold)[0] - det_accu = det[merge_index, :] - det = np.delete(det, merge_index, 0) - if merge_index.shape[0] <= 1: - if det.shape[0] == 0: - try: - dets = np.row_stack((dets, det_accu)) - except: - dets = det_accu - continue - det_accu[:, 0:4] = det_accu[:, 0:4] * np.tile(det_accu[:, -1:], (1, 4)) - max_score = np.max(det_accu[:, 4]) - det_accu_sum = np.zeros((1, 5)) - det_accu_sum[:, 0:4] = np.sum(det_accu[:, 0:4], - axis=0) / np.sum(det_accu[:, -1:]) - det_accu_sum[:, 4] = max_score - if dets is None: - dets = det_accu_sum - else: - dets = np.row_stack((dets, det_accu_sum)) - dets = dets[0:750, :] - return dets - diff --git a/RetinaFaceAntiCov/test.py b/RetinaFaceAntiCov/test.py deleted file mode 100644 index be177dc..0000000 --- a/RetinaFaceAntiCov/test.py +++ /dev/null @@ -1,65 +0,0 @@ -import cv2 -import sys -import numpy as np -import datetime -import os -import glob -from retinaface_cov import RetinaFaceCoV - -thresh = 0.8 -mask_thresh = 0.2 -scales = [640, 1080] - -count = 1 - -gpuid = 0 -#detector = RetinaFaceCoV('./model/mnet_cov1', 0, gpuid, 'net3') -detector = RetinaFaceCoV('./model/mnet_cov2', 0, gpuid, 'net3l') - -img = cv2.imread('n1.jpg') -print(img.shape) -im_shape = img.shape -target_size = scales[0] -max_size = scales[1] -im_size_min = np.min(im_shape[0:2]) -im_size_max = np.max(im_shape[0:2]) -#im_scale = 1.0 -#if im_size_min>target_size or im_size_max>max_size: -im_scale = float(target_size) / float(im_size_min) -# prevent bigger axis from being more than max_size: -if np.round(im_scale * im_size_max) > max_size: - im_scale = float(max_size) / float(im_size_max) - -print('im_scale', im_scale) - -scales = [im_scale] -flip = False - -for c in range(count): - faces, landmarks = detector.detect(img, thresh, scales=scales, do_flip=flip) - - -if faces is not None: - print('find', faces.shape[0], 'faces') - for i in range(faces.shape[0]): - #print('score', faces[i][4]) - face = faces[i] - box = face[0:4].astype(np.int) - mask = face[5] - print(i,box,mask) - #color = (255,0,0) - if mask>=mask_thresh: - color = (0,0,255) - else: - color = (0,255,0) - cv2.rectangle(img, (box[0], box[1]), (box[2], box[3]), color, 2) - landmark5 = landmarks[i].astype(np.int) - #print(landmark.shape) - for l in range(landmark5.shape[0]): - color = (255,0,0) - cv2.circle(img, (landmark5[l][0], landmark5[l][1]), 1, color, 2) - - filename = './cov_test.jpg' - print('writing', filename) - cv2.imwrite(filename, img) - diff --git a/alignment/coordinateReg/image_infer.py b/alignment/coordinateReg/image_infer.py index 3b2fa6c..96099a4 100644 --- a/alignment/coordinateReg/image_infer.py +++ b/alignment/coordinateReg/image_infer.py @@ -8,35 +8,41 @@ import datetime from skimage import transform as trans import insightface + def square_crop(im, S): - if im.shape[0]>im.shape[1]: - height = S - width = int( float(im.shape[1]) / im.shape[0] * S ) - scale = float(S) / im.shape[0] - else: - width = S - height = int( float(im.shape[0]) / im.shape[1] * S ) - scale = float(S) / im.shape[1] - resized_im = cv2.resize(im, (width, height)) - det_im = np.zeros( (S, S, 3), dtype=np.uint8 ) - det_im[:resized_im.shape[0], :resized_im.shape[1], :] = resized_im - return det_im, scale + if im.shape[0] > im.shape[1]: + height = S + width = int(float(im.shape[1]) / im.shape[0] * S) + scale = float(S) / im.shape[0] + else: + width = S + height = int(float(im.shape[0]) / im.shape[1] * S) + scale = float(S) / im.shape[1] + resized_im = cv2.resize(im, (width, height)) + det_im = np.zeros((S, S, 3), dtype=np.uint8) + det_im[:resized_im.shape[0], :resized_im.shape[1], :] = resized_im + return det_im, scale + def transform(data, center, output_size, scale, rotation): scale_ratio = scale - rot = float(rotation)*np.pi/180.0 + rot = float(rotation) * np.pi / 180.0 #translation = (output_size/2-center[0]*scale_ratio, output_size/2-center[1]*scale_ratio) t1 = trans.SimilarityTransform(scale=scale_ratio) - cx = center[0]*scale_ratio - cy = center[1]*scale_ratio - t2 = trans.SimilarityTransform(translation=(-1*cx, -1*cy)) + cx = center[0] * scale_ratio + cy = center[1] * scale_ratio + t2 = trans.SimilarityTransform(translation=(-1 * cx, -1 * cy)) t3 = trans.SimilarityTransform(rotation=rot) - t4 = trans.SimilarityTransform(translation=(output_size/2, output_size/2)) - t = t1+t2+t3+t4 + t4 = trans.SimilarityTransform(translation=(output_size / 2, + output_size / 2)) + t = t1 + t2 + t3 + t4 M = t.params[0:2] - cropped = cv2.warpAffine(data,M,(output_size, output_size), borderValue = 0.0) + cropped = cv2.warpAffine(data, + M, (output_size, output_size), + borderValue=0.0) return cropped, M + def trans_points2d(pts, M): new_pts = np.zeros(shape=pts.shape, dtype=np.float32) for i in range(pts.shape[0]): @@ -48,8 +54,9 @@ def trans_points2d(pts, M): return new_pts + def trans_points3d(pts, M): - scale = np.sqrt(M[0][0]*M[0][0] + M[0][1]*M[0][1]) + scale = np.sqrt(M[0][0] * M[0][0] + M[0][1] * M[0][1]) #print(scale) new_pts = np.zeros(shape=pts.shape, dtype=np.float32) for i in range(pts.shape[0]): @@ -58,97 +65,98 @@ def trans_points3d(pts, M): new_pt = np.dot(M, new_pt) #print('new_pt', new_pt.shape, new_pt) new_pts[i][0:2] = new_pt[0:2] - new_pts[i][2] = pts[i][2]*scale + new_pts[i][2] = pts[i][2] * scale return new_pts + def trans_points(pts, M): - if pts.shape[1]==2: - return trans_points2d(pts, M) - else: - return trans_points3d(pts, M) + if pts.shape[1] == 2: + return trans_points2d(pts, M) + else: + return trans_points3d(pts, M) class Handler: - def __init__(self, prefix, epoch, im_size=192, det_size=224, ctx_id=0): - print('loading',prefix, epoch) - if ctx_id>=0: - ctx = mx.gpu(ctx_id) - else: - ctx = mx.cpu() - image_size = (im_size, im_size) - self.detector = insightface.model_zoo.get_model('retinaface_mnet025_v2') #can replace with your own face detector - #self.detector = insightface.model_zoo.get_model('retinaface_r50_v1') - self.detector.prepare(ctx_id=ctx_id) - self.det_size = det_size - sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) - all_layers = sym.get_internals() - sym = all_layers['fc1_output'] - self.image_size = image_size - model = mx.mod.Module(symbol=sym, context=ctx, label_names = None) - model.bind(for_training=False, data_shapes=[('data', (1, 3, image_size[0], image_size[1]))]) - model.set_params(arg_params, aux_params) - self.model = model - self.image_size = image_size + def __init__(self, prefix, epoch, im_size=192, det_size=224, ctx_id=0): + print('loading', prefix, epoch) + if ctx_id >= 0: + ctx = mx.gpu(ctx_id) + else: + ctx = mx.cpu() + image_size = (im_size, im_size) + self.detector = insightface.model_zoo.get_model( + 'retinaface_mnet025_v2') #can replace with your own face detector + #self.detector = insightface.model_zoo.get_model('retinaface_r50_v1') + self.detector.prepare(ctx_id=ctx_id) + self.det_size = det_size + sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) + all_layers = sym.get_internals() + sym = all_layers['fc1_output'] + self.image_size = image_size + model = mx.mod.Module(symbol=sym, context=ctx, label_names=None) + model.bind(for_training=False, + data_shapes=[('data', (1, 3, image_size[0], image_size[1])) + ]) + model.set_params(arg_params, aux_params) + self.model = model + self.image_size = image_size + def get(self, img, get_all=False): + out = [] + det_im, det_scale = square_crop(img, self.det_size) + bboxes, _ = self.detector.detect(det_im) + if bboxes.shape[0] == 0: + return out + bboxes /= det_scale + if not get_all: + areas = [] + for i in range(bboxes.shape[0]): + x = bboxes[i] + area = (x[2] - x[0]) * (x[3] - x[1]) + areas.append(area) + m = np.argsort(areas)[-1] + bboxes = bboxes[m:m + 1] + for i in range(bboxes.shape[0]): + bbox = bboxes[i] + input_blob = np.zeros((1, 3) + self.image_size, dtype=np.float32) + w, h = (bbox[2] - bbox[0]), (bbox[3] - bbox[1]) + center = (bbox[2] + bbox[0]) / 2, (bbox[3] + bbox[1]) / 2 + rotate = 0 + _scale = self.image_size[0] * 2 / 3.0 / max(w, h) + rimg, M = transform(img, center, self.image_size[0], _scale, + rotate) + rimg = cv2.cvtColor(rimg, cv2.COLOR_BGR2RGB) + rimg = np.transpose(rimg, (2, 0, 1)) #3*112*112, RGB + input_blob[0] = rimg + data = mx.nd.array(input_blob) + db = mx.io.DataBatch(data=(data, )) + self.model.forward(db, is_train=False) + pred = self.model.get_outputs()[-1].asnumpy()[0] + if pred.shape[0] >= 3000: + pred = pred.reshape((-1, 3)) + else: + pred = pred.reshape((-1, 2)) + pred[:, 0:2] += 1 + pred[:, 0:2] *= (self.image_size[0] // 2) + if pred.shape[1] == 3: + pred[:, 2] *= (self.image_size[0] // 2) - - def get(self, img, get_all=False): - out = [] - det_im, det_scale = square_crop(img, self.det_size) - bboxes, _ = self.detector.detect(det_im) - if bboxes.shape[0]==0: + IM = cv2.invertAffineTransform(M) + pred = trans_points(pred, IM) + out.append(pred) return out - bboxes /= det_scale - if not get_all: - areas = [] - for i in range(bboxes.shape[0]): - x = bboxes[i] - area = (x[2]-x[0])*(x[3]-x[1]) - areas.append(area) - m = np.argsort(areas)[-1] - bboxes = bboxes[m:m+1] - for i in range(bboxes.shape[0]): - bbox = bboxes[i] - input_blob = np.zeros( (1, 3)+self.image_size,dtype=np.float32) - w, h = (bbox[2]-bbox[0]), (bbox[3]-bbox[1]) - center = (bbox[2]+bbox[0])/2, (bbox[3]+bbox[1])/2 - rotate = 0 - _scale = self.image_size[0]*2/3.0/max(w,h) - rimg, M = transform(img, center, self.image_size[0], _scale, rotate) - rimg = cv2.cvtColor(rimg, cv2.COLOR_BGR2RGB) - rimg = np.transpose(rimg, (2,0,1)) #3*112*112, RGB - input_blob[0] = rimg - data = mx.nd.array(input_blob) - db = mx.io.DataBatch(data=(data,)) - self.model.forward(db, is_train=False) - pred = self.model.get_outputs()[-1].asnumpy()[0] - if pred.shape[0]>=3000: - pred = pred.reshape( (-1, 3) ) - else: - pred = pred.reshape( (-1, 2) ) - pred[:,0:2] += 1 - pred[:,0:2] *= (self.image_size[0]//2) - if pred.shape[1]==3: - pred[:,2] *= (self.image_size[0]//2) - - IM = cv2.invertAffineTransform(M) - pred = trans_points(pred, IM) - out.append(pred) - return out if __name__ == '__main__': - handler = Handler('./model/2d106_det', 0, ctx_id=7, det_size=640) - im = cv2.imread('../../sample-images/t1.jpg') - tim = im.copy() - preds = handler.get(im, get_all=True) - color = (200, 160, 75) - for pred in preds: - pred = np.round(pred).astype(np.int) - for i in range(pred.shape[0]): - p = tuple(pred[i]) - cv2.circle(tim, p, 1, color, 1,cv2.LINE_AA) - cv2.imwrite('./test_out.jpg', tim) - - + handler = Handler('./model/2d106_det', 0, ctx_id=7, det_size=640) + im = cv2.imread('../../sample-images/t1.jpg') + tim = im.copy() + preds = handler.get(im, get_all=True) + color = (200, 160, 75) + for pred in preds: + pred = np.round(pred).astype(np.int) + for i in range(pred.shape[0]): + p = tuple(pred[i]) + cv2.circle(tim, p, 1, color, 1, cv2.LINE_AA) + cv2.imwrite('./test_out.jpg', tim) diff --git a/alignment/heatmapReg/data.py b/alignment/heatmapReg/data.py index de6bcb0..eeaca9c 100644 --- a/alignment/heatmapReg/data.py +++ b/alignment/heatmapReg/data.py @@ -18,97 +18,103 @@ from PIL import Image from config import config from skimage import transform as tf + class FaceSegIter(DataIter): - def __init__(self, batch_size, - per_batch_size = 0, - path_imgrec = None, - aug_level = 0, - force_mirror = False, - exf = 1, - use_coherent = 0, - args = None, - data_name = "data", - label_name = "softmax_label"): - self.aug_level = aug_level - self.force_mirror = force_mirror - self.use_coherent = use_coherent - self.exf = exf - self.batch_size = batch_size - self.per_batch_size = per_batch_size - self.data_name = data_name - self.label_name = label_name - assert path_imgrec - logging.info('loading recordio %s...', - path_imgrec) - path_imgidx = path_imgrec[0:-4]+".idx" - self.imgrec = mx.recordio.MXIndexedRecordIO(path_imgidx, path_imgrec, 'r') # pylint: disable=redefined-variable-type - self.oseq = list(self.imgrec.keys) - print('train size', len(self.oseq)) - self.cur = 0 - self.reset() - self.data_shape = (3, config.input_img_size, config.input_img_size) - self.num_classes = config.num_classes - self.input_img_size = config.input_img_size - #self.label_classes = self.num_classes - if config.losstype=='heatmap': - if aug_level>0: - self.output_label_size = config.output_label_size - self.label_shape = (self.num_classes, self.output_label_size, self.output_label_size) + def __init__(self, + batch_size, + per_batch_size=0, + path_imgrec=None, + aug_level=0, + force_mirror=False, + exf=1, + use_coherent=0, + args=None, + data_name="data", + label_name="softmax_label"): + self.aug_level = aug_level + self.force_mirror = force_mirror + self.use_coherent = use_coherent + self.exf = exf + self.batch_size = batch_size + self.per_batch_size = per_batch_size + self.data_name = data_name + self.label_name = label_name + assert path_imgrec + logging.info('loading recordio %s...', path_imgrec) + path_imgidx = path_imgrec[0:-4] + ".idx" + self.imgrec = mx.recordio.MXIndexedRecordIO(path_imgidx, path_imgrec, + 'r') # pylint: disable=redefined-variable-type + self.oseq = list(self.imgrec.keys) + print('train size', len(self.oseq)) + self.cur = 0 + self.reset() + self.data_shape = (3, config.input_img_size, config.input_img_size) + self.num_classes = config.num_classes + self.input_img_size = config.input_img_size + #self.label_classes = self.num_classes + if config.losstype == 'heatmap': + if aug_level > 0: + self.output_label_size = config.output_label_size + self.label_shape = (self.num_classes, self.output_label_size, + self.output_label_size) + else: + self.output_label_size = self.input_img_size + #self.label_shape = (self.num_classes, 2) + self.label_shape = (self.num_classes, self.output_label_size, + self.output_label_size) else: - self.output_label_size = self.input_img_size - #self.label_shape = (self.num_classes, 2) - self.label_shape = (self.num_classes, self.output_label_size, self.output_label_size) - else: - if aug_level>0: - self.output_label_size = config.output_label_size - self.label_shape = (self.num_classes, 2) - else: - self.output_label_size = self.input_img_size - #self.label_shape = (self.num_classes, 2) - self.label_shape = (self.num_classes, 2) - self.provide_data = [(data_name, (batch_size,) + self.data_shape)] - self.provide_label = [(label_name, (batch_size,) + self.label_shape)] - self.img_num = 0 - self.invalid_num = 0 - self.mode = 1 - self.vis = 0 - self.stats = [0,0] - self.flip_order = [16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, - 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 27, 28, 29, 30, 35, 34, 33, 32, 31, - 45, 44, 43, 42, 47, 46, 39, 38, 37, 36, 41, 40, 54, 53, 52, 51, 50, 49, 48, - 59, 58, 57, 56, 55, 64, 63, 62, 61, 60, 67, 66, 65] - #self.mirror_set = [ - # (22,23), - # (21,24), - # (20,25), - # (19,26), - # (18,27), - # (40,43), - # (39,44), - # (38,45), - # (37,46), - # (42,47), - # (41,48), - # (33,35), - # (32,36), - # (51,53), - # (50,54), - # (62,64), - # (61,65), - # (49,55), - # (49,55), - # (68,66), - # (60,56), - # (59,57), - # (1,17), - # (2,16), - # (3,15), - # (4,14), - # (5,13), - # (6,12), - # (7,11), - # (8,10), - # ] + if aug_level > 0: + self.output_label_size = config.output_label_size + self.label_shape = (self.num_classes, 2) + else: + self.output_label_size = self.input_img_size + #self.label_shape = (self.num_classes, 2) + self.label_shape = (self.num_classes, 2) + self.provide_data = [(data_name, (batch_size, ) + self.data_shape)] + self.provide_label = [(label_name, (batch_size, ) + self.label_shape)] + self.img_num = 0 + self.invalid_num = 0 + self.mode = 1 + self.vis = 0 + self.stats = [0, 0] + self.flip_order = [ + 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 26, 25, + 24, 23, 22, 21, 20, 19, 18, 17, 27, 28, 29, 30, 35, 34, 33, 32, 31, + 45, 44, 43, 42, 47, 46, 39, 38, 37, 36, 41, 40, 54, 53, 52, 51, 50, + 49, 48, 59, 58, 57, 56, 55, 64, 63, 62, 61, 60, 67, 66, 65 + ] + #self.mirror_set = [ + # (22,23), + # (21,24), + # (20,25), + # (19,26), + # (18,27), + # (40,43), + # (39,44), + # (38,45), + # (37,46), + # (42,47), + # (41,48), + # (33,35), + # (32,36), + # (51,53), + # (50,54), + # (62,64), + # (61,65), + # (49,55), + # (49,55), + # (68,66), + # (60,56), + # (59,57), + # (1,17), + # (2,16), + # (3,15), + # (4,14), + # (5,13), + # (6,12), + # (7,11), + # (8,10), + # ] def get_data_shape(self): return self.data_shape @@ -118,181 +124,190 @@ class FaceSegIter(DataIter): def get_shape_dict(self): D = {} - for (k,v) in self.provide_data: + for (k, v) in self.provide_data: D[k] = v - for (k,v) in self.provide_label: + for (k, v) in self.provide_label: D[k] = v return D def get_label_names(self): D = [] - for (k,v) in self.provide_label: + for (k, v) in self.provide_label: D.append(k) return D def reset(self): - #print('reset') - if self.aug_level==0: - self.seq = self.oseq - else: - self.seq = [] - for _ in range(self.exf): - _seq = self.oseq[:] - random.shuffle(_seq) - self.seq += _seq - print('train size after reset', len(self.seq)) - self.cur = 0 + #print('reset') + if self.aug_level == 0: + self.seq = self.oseq + else: + self.seq = [] + for _ in range(self.exf): + _seq = self.oseq[:] + random.shuffle(_seq) + self.seq += _seq + print('train size after reset', len(self.seq)) + self.cur = 0 def next_sample(self): - """Helper function for reading in next sample.""" - if self.cur >= len(self.seq): - raise StopIteration - idx = self.seq[self.cur] - self.cur += 1 - s = self.imgrec.read_idx(idx) - header, img = recordio.unpack(s) - img = mx.image.imdecode(img).asnumpy() - hlabel = np.array(header.label).reshape( (self.num_classes,2) ) - if not config.label_xfirst: - hlabel = hlabel[:,::-1] #convert to X/W first - annot = {'scale': config.base_scale} + """Helper function for reading in next sample.""" + if self.cur >= len(self.seq): + raise StopIteration + idx = self.seq[self.cur] + self.cur += 1 + s = self.imgrec.read_idx(idx) + header, img = recordio.unpack(s) + img = mx.image.imdecode(img).asnumpy() + hlabel = np.array(header.label).reshape((self.num_classes, 2)) + if not config.label_xfirst: + hlabel = hlabel[:, ::-1] #convert to X/W first + annot = {'scale': config.base_scale} - #ul = np.array( (50000,50000), dtype=np.int32) - #br = np.array( (0,0), dtype=np.int32) - #for i in range(hlabel.shape[0]): - # h = int(hlabel[i][0]) - # w = int(hlabel[i][1]) - # key = np.array((h,w)) - # ul = np.minimum(key, ul) - # br = np.maximum(key, br) + #ul = np.array( (50000,50000), dtype=np.int32) + #br = np.array( (0,0), dtype=np.int32) + #for i in range(hlabel.shape[0]): + # h = int(hlabel[i][0]) + # w = int(hlabel[i][1]) + # key = np.array((h,w)) + # ul = np.minimum(key, ul) + # br = np.maximum(key, br) - return img, hlabel, annot + return img, hlabel, annot def get_flip(self, data, label): - data_flip = np.zeros_like(data) - label_flip = np.zeros_like(label) - for k in range(data_flip.shape[2]): - data_flip[:,:,k] = np.fliplr(data[:,:,k]) - for k in range(label_flip.shape[0]): - label_flip[k,:] = np.fliplr(label[k,:]) - #print(label[0,:].shape) - label_flip = label_flip[self.flip_order,:] - return data_flip, label_flip + data_flip = np.zeros_like(data) + label_flip = np.zeros_like(label) + for k in range(data_flip.shape[2]): + data_flip[:, :, k] = np.fliplr(data[:, :, k]) + for k in range(label_flip.shape[0]): + label_flip[k, :] = np.fliplr(label[k, :]) + #print(label[0,:].shape) + label_flip = label_flip[self.flip_order, :] + return data_flip, label_flip def get_data(self, data, label, annot): - if self.vis: - self.img_num+=1 - #if self.img_num<=self.vis: - # filename = './vis/raw_%d.jpg' % (self.img_num) - # print('save', filename) - # draw = data.copy() - # for i in range(label.shape[0]): - # cv2.circle(draw, (label[i][1], label[i][0]), 1, (0, 0, 255), 2) - # scipy.misc.imsave(filename, draw) + if self.vis: + self.img_num += 1 + #if self.img_num<=self.vis: + # filename = './vis/raw_%d.jpg' % (self.img_num) + # print('save', filename) + # draw = data.copy() + # for i in range(label.shape[0]): + # cv2.circle(draw, (label[i][1], label[i][0]), 1, (0, 0, 255), 2) + # scipy.misc.imsave(filename, draw) - rotate = 0 - #scale = 1.0 - if 'scale' in annot: - scale = annot['scale'] - else: - scale = max(data.shape[0], data.shape[1]) - if 'center' in annot: - center = annot['center'] - else: - center = np.array( (data.shape[1]/2, data.shape[0]/2) ) - max_retry = 3 - if self.aug_level==0: #validation mode - max_retry = 6 - retry = 0 - found = False - base_scale = scale - while retry0: - rotate = np.random.randint(-40, 40) - scale_config = 0.2 - #rotate = 0 - #scale_config = 0.0 - scale_ratio = min(1+scale_config, max(1-scale_config, (np.random.randn() * scale_config) + 1)) - _scale = int(base_scale * scale_ratio) - #translate = np.random.randint(-5, 5, size=(2,)) - #center += translate - data_out, trans = img_helper.transform(data, center, self.input_img_size, _scale, rotate) - #data_out = img_helper.crop2(data, center, _scale, (self.input_img_size, self.input_img_size), rot=rotate) - label_out = np.zeros(self.label_shape, dtype=np.float32) - #print('out shapes', data_out.shape, label_out.shape) - for i in range(label.shape[0]): - pt = label[i].copy() - #pt = pt[::-1] - npt = img_helper.transform_pt(pt, trans) - if npt[0]>=data_out.shape[1] or npt[1]>=data_out.shape[0] or npt[0]<0 or npt[1]<0: - succ = False - #print('err npt', npt) - break - if config.losstype=='heatmap': - pt_scale = float(self.output_label_size)/self.input_img_size - npt *= pt_scale - npt = npt.astype(np.int32) - img_helper.gaussian(label_out[i], npt, config.gaussian) - else: - label_out[i] = (npt/self.input_img_size) - #print('before gaussian', label_out[i].shape, pt.shape) - #trans = img_helper.transform(pt, center, _scale, (self.output_label_size, self.output_label_size), rot=rotate) - #print(trans.shape) - #if not img_helper.gaussian(label_out[i], trans, _g): - # succ = False - # break - if not succ: - if self.aug_level==0: - base_scale+=20 - continue - - flip_data_out = None - flip_label_out = None - if config.net_coherent: - flip_data_out, flip_label_out = self.get_flip(data_out, label_out) - elif ((self.aug_level>0 and np.random.rand() < 0.5) or self.force_mirror): #flip aug - flip_data_out, flip_label_out = self.get_flip(data_out, label_out) - data_out, label_out = flip_data_out, flip_label_out + rotate = 0 + #scale = 1.0 + if 'scale' in annot: + scale = annot['scale'] + else: + scale = max(data.shape[0], data.shape[1]) + if 'center' in annot: + center = annot['center'] + else: + center = np.array((data.shape[1] / 2, data.shape[0] / 2)) + max_retry = 3 + if self.aug_level == 0: #validation mode + max_retry = 6 + retry = 0 + found = False + base_scale = scale + while retry < max_retry: + retry += 1 + succ = True + _scale = base_scale + if self.aug_level > 0: + rotate = np.random.randint(-40, 40) + scale_config = 0.2 + #rotate = 0 + #scale_config = 0.0 + scale_ratio = min( + 1 + scale_config, + max(1 - scale_config, + (np.random.randn() * scale_config) + 1)) + _scale = int(base_scale * scale_ratio) + #translate = np.random.randint(-5, 5, size=(2,)) + #center += translate + data_out, trans = img_helper.transform(data, center, + self.input_img_size, _scale, + rotate) + #data_out = img_helper.crop2(data, center, _scale, (self.input_img_size, self.input_img_size), rot=rotate) + label_out = np.zeros(self.label_shape, dtype=np.float32) + #print('out shapes', data_out.shape, label_out.shape) + for i in range(label.shape[0]): + pt = label[i].copy() + #pt = pt[::-1] + npt = img_helper.transform_pt(pt, trans) + if npt[0] >= data_out.shape[1] or npt[1] >= data_out.shape[ + 0] or npt[0] < 0 or npt[1] < 0: + succ = False + #print('err npt', npt) + break + if config.losstype == 'heatmap': + pt_scale = float( + self.output_label_size) / self.input_img_size + npt *= pt_scale + npt = npt.astype(np.int32) + img_helper.gaussian(label_out[i], npt, config.gaussian) + else: + label_out[i] = (npt / self.input_img_size) + #print('before gaussian', label_out[i].shape, pt.shape) + #trans = img_helper.transform(pt, center, _scale, (self.output_label_size, self.output_label_size), rot=rotate) + #print(trans.shape) + #if not img_helper.gaussian(label_out[i], trans, _g): + # succ = False + # break + if not succ: + if self.aug_level == 0: + base_scale += 20 + continue - found = True - break + flip_data_out = None + flip_label_out = None + if config.net_coherent: + flip_data_out, flip_label_out = self.get_flip( + data_out, label_out) + elif ((self.aug_level > 0 and np.random.rand() < 0.5) + or self.force_mirror): #flip aug + flip_data_out, flip_label_out = self.get_flip( + data_out, label_out) + data_out, label_out = flip_data_out, flip_label_out + found = True + break - #self.stats[0]+=1 - if not found: - #self.stats[1]+=1 - #print('find aug error', retry) - #print(self.stats) - #print('!!!ERR') - return None - #print('found with scale', _scale, rotate) + #self.stats[0]+=1 + if not found: + #self.stats[1]+=1 + #print('find aug error', retry) + #print(self.stats) + #print('!!!ERR') + return None + #print('found with scale', _scale, rotate) + if self.vis > 0 and self.img_num <= self.vis: + print('crop', data.shape, center, _scale, rotate, data_out.shape) + filename = './vis/cropped_%d.jpg' % (self.img_num) + print('save', filename) + draw = data_out.copy() + alabel = label_out.copy() + for i in range(label.shape[0]): + a = cv2.resize(alabel[i], + (self.input_img_size, self.input_img_size)) + ind = np.unravel_index(np.argmax(a, axis=None), a.shape) + cv2.circle(draw, (ind[1], ind[0]), 1, (0, 0, 255), 2) + scipy.misc.imsave(filename, draw) + filename = './vis/raw_%d.jpg' % (self.img_num) + scipy.misc.imsave(filename, data) - if self.vis>0 and self.img_num<=self.vis: - print('crop', data.shape, center, _scale, rotate, data_out.shape) - filename = './vis/cropped_%d.jpg' % (self.img_num) - print('save', filename) - draw = data_out.copy() - alabel = label_out.copy() - for i in range(label.shape[0]): - a = cv2.resize(alabel[i], (self.input_img_size, self.input_img_size)) - ind = np.unravel_index(np.argmax(a, axis=None), a.shape) - cv2.circle(draw, (ind[1], ind[0]), 1, (0, 0, 255), 2) - scipy.misc.imsave(filename, draw) - filename = './vis/raw_%d.jpg' % (self.img_num) - scipy.misc.imsave(filename, data) - - return data_out, label_out, flip_data_out,flip_label_out + return data_out, label_out, flip_data_out, flip_label_out def next(self): """Returns the next batch of data.""" #print('next') batch_size = self.batch_size - batch_data = nd.empty((batch_size,)+self.data_shape) - batch_label = nd.empty((batch_size,)+self.label_shape) + batch_data = nd.empty((batch_size, ) + self.data_shape) + batch_label = nd.empty((batch_size, ) + self.label_shape) i = 0 #self.cutoff = random.randint(800,1280) try: @@ -301,7 +316,7 @@ class FaceSegIter(DataIter): data, label, annot = self.next_sample() R = self.get_data(data, label, annot) if R is None: - continue + continue data_out, label_out, flip_data_out, flip_label_out = R if not self.use_coherent: data = nd.array(data_out) @@ -323,18 +338,17 @@ class FaceSegIter(DataIter): batch_data[i][:] = data batch_label[i][:] = label #i+=1 - j = i+self.per_batch_size//2 + j = i + self.per_batch_size // 2 batch_data[j][:] = data2 batch_label[j][:] = label2 i += 1 - if j%self.per_batch_size==self.per_batch_size-1: - i = j+1 + if j % self.per_batch_size == self.per_batch_size - 1: + i = j + 1 except StopIteration: - if i=0) - if sigma==0: - img[pt[1], pt[0]] = 1.0 - return True + assert (sigma >= 0) + if sigma == 0: + img[pt[1], pt[0]] = 1.0 + return True #assert pt[0]<=img.shape[1] #assert pt[1]<=img.shape[0] # Check that any part of the gaussian is in-bounds ul = [int(pt[0] - 3 * sigma), int(pt[1] - 3 * sigma)] br = [int(pt[0] + 3 * sigma + 1), int(pt[1] + 3 * sigma + 1)] - if (ul[0] > img.shape[1] or ul[1] >= img.shape[0] or - br[0] < 0 or br[1] < 0): + if (ul[0] > img.shape[1] or ul[1] >= img.shape[0] or br[0] < 0 + or br[1] < 0): # If not, just return the image as is #print('gaussian error') return False @@ -50,7 +56,7 @@ def gaussian(img, pt, sigma): y = x[:, np.newaxis] x0 = y0 = size // 2 # The gaussian is not normalized, we want the center value to equal 1 - g = np.exp(- ((x - x0) ** 2 + (y - y0) ** 2) / (2 * sigma ** 2)) + g = np.exp(-((x - x0)**2 + (y - y0)**2) / (2 * sigma**2)) # Usable gaussian range g_x = max(0, -ul[0]), min(br[0], img.shape[1]) - ul[0] @@ -63,18 +69,18 @@ def gaussian(img, pt, sigma): return True #return img -def estimate_trans_bbox(face, input_size, s = 2.0): - w = face[2] - face[0] - h = face[3] - face[1] - wc = int( (face[2]+face[0])/2 ) - hc = int( (face[3]+face[1])/2 ) - im_size = max(w, h) - #size = int(im_size*1.2) - scale = input_size/(max(w,h)*s) - M = [ - [scale, 0, input_size/2-wc*scale], - [0, scale, input_size/2-hc*scale], - ] - M = np.array(M) - return M +def estimate_trans_bbox(face, input_size, s=2.0): + w = face[2] - face[0] + h = face[3] - face[1] + wc = int((face[2] + face[0]) / 2) + hc = int((face[3] + face[1]) / 2) + im_size = max(w, h) + #size = int(im_size*1.2) + scale = input_size / (max(w, h) * s) + M = [ + [scale, 0, input_size / 2 - wc * scale], + [0, scale, input_size / 2 - hc * scale], + ] + M = np.array(M) + return M diff --git a/alignment/heatmapReg/metric.py b/alignment/heatmapReg/metric.py index 16ffe1c..2ddc96c 100644 --- a/alignment/heatmapReg/metric.py +++ b/alignment/heatmapReg/metric.py @@ -4,96 +4,104 @@ import math import cv2 from config import config -class LossValueMetric(mx.metric.EvalMetric): - def __init__(self): - self.axis = 1 - super(LossValueMetric, self).__init__( - 'lossvalue', axis=self.axis, - output_names=None, label_names=None) - self.losses = [] - def update(self, labels, preds): - loss = preds[0].asnumpy()[0] - self.sum_metric += loss - self.num_inst += 1.0 +class LossValueMetric(mx.metric.EvalMetric): + def __init__(self): + self.axis = 1 + super(LossValueMetric, self).__init__('lossvalue', + axis=self.axis, + output_names=None, + label_names=None) + self.losses = [] + + def update(self, labels, preds): + loss = preds[0].asnumpy()[0] + self.sum_metric += loss + self.num_inst += 1.0 + class NMEMetric(mx.metric.EvalMetric): - def __init__(self): - self.axis = 1 - super(NMEMetric, self).__init__( - 'NME', axis=self.axis, - output_names=None, label_names=None) - #self.losses = [] - self.count = 0 + def __init__(self): + self.axis = 1 + super(NMEMetric, self).__init__('NME', + axis=self.axis, + output_names=None, + label_names=None) + #self.losses = [] + self.count = 0 - def cal_nme(self, label, pred_label): - nme = [] - for b in range(pred_label.shape[0]): - record = [None]*6 - item = [] - if label.ndim==4: - _heatmap = label[b][36] - if np.count_nonzero(_heatmap)==0: - continue - else:#ndim==3 - #print(label[b]) - if np.count_nonzero(label[b])==0: - continue - for p in range(pred_label.shape[1]): - if label.ndim==4: - heatmap_gt = label[b][p] - ind_gt = np.unravel_index(np.argmax(heatmap_gt, axis=None), heatmap_gt.shape) - ind_gt = np.array(ind_gt) - else: - ind_gt = label[b][p] - #ind_gt = ind_gt.astype(np.int) - #print(ind_gt) - heatmap_pred = pred_label[b][p] - heatmap_pred = cv2.resize(heatmap_pred, (config.input_img_size, config.input_img_size)) - ind_pred = np.unravel_index(np.argmax(heatmap_pred, axis=None), heatmap_pred.shape) - ind_pred = np.array(ind_pred) - #print(ind_gt.shape) - #print(ind_pred) - if p==36: - #print('b', b, p, ind_gt, np.count_nonzero(heatmap_gt)) - record[0] = ind_gt - elif p==39: - record[1] = ind_gt - elif p==42: - record[2] = ind_gt - elif p==45: - record[3] = ind_gt - if record[4] is None or record[5] is None: - record[4] = ind_gt - record[5] = ind_gt - else: - record[4] = np.minimum(record[4], ind_gt) - record[5] = np.maximum(record[5], ind_gt) - #print(ind_gt.shape, ind_pred.shape) - value = np.sqrt(np.sum(np.square(ind_gt - ind_pred))) - item.append(value) - _nme = np.mean(item) - if config.landmark_type=='2d': - left_eye = (record[0]+record[1])/2 - right_eye = (record[2]+record[3])/2 - _dist = np.sqrt(np.sum(np.square(left_eye - right_eye))) - #print('eye dist', _dist, left_eye, right_eye) - _nme /= _dist - else: - #_dist = np.sqrt(float(label.shape[2]*label.shape[3])) - _dist = np.sqrt(np.sum(np.square(record[5] - record[4]))) - #print(_dist) - _nme /= _dist - nme.append(_nme) - return np.mean(nme) + def cal_nme(self, label, pred_label): + nme = [] + for b in range(pred_label.shape[0]): + record = [None] * 6 + item = [] + if label.ndim == 4: + _heatmap = label[b][36] + if np.count_nonzero(_heatmap) == 0: + continue + else: #ndim==3 + #print(label[b]) + if np.count_nonzero(label[b]) == 0: + continue + for p in range(pred_label.shape[1]): + if label.ndim == 4: + heatmap_gt = label[b][p] + ind_gt = np.unravel_index(np.argmax(heatmap_gt, axis=None), + heatmap_gt.shape) + ind_gt = np.array(ind_gt) + else: + ind_gt = label[b][p] + #ind_gt = ind_gt.astype(np.int) + #print(ind_gt) + heatmap_pred = pred_label[b][p] + heatmap_pred = cv2.resize( + heatmap_pred, + (config.input_img_size, config.input_img_size)) + ind_pred = np.unravel_index(np.argmax(heatmap_pred, axis=None), + heatmap_pred.shape) + ind_pred = np.array(ind_pred) + #print(ind_gt.shape) + #print(ind_pred) + if p == 36: + #print('b', b, p, ind_gt, np.count_nonzero(heatmap_gt)) + record[0] = ind_gt + elif p == 39: + record[1] = ind_gt + elif p == 42: + record[2] = ind_gt + elif p == 45: + record[3] = ind_gt + if record[4] is None or record[5] is None: + record[4] = ind_gt + record[5] = ind_gt + else: + record[4] = np.minimum(record[4], ind_gt) + record[5] = np.maximum(record[5], ind_gt) + #print(ind_gt.shape, ind_pred.shape) + value = np.sqrt(np.sum(np.square(ind_gt - ind_pred))) + item.append(value) + _nme = np.mean(item) + if config.landmark_type == '2d': + left_eye = (record[0] + record[1]) / 2 + right_eye = (record[2] + record[3]) / 2 + _dist = np.sqrt(np.sum(np.square(left_eye - right_eye))) + #print('eye dist', _dist, left_eye, right_eye) + _nme /= _dist + else: + #_dist = np.sqrt(float(label.shape[2]*label.shape[3])) + _dist = np.sqrt(np.sum(np.square(record[5] - record[4]))) + #print(_dist) + _nme /= _dist + nme.append(_nme) + return np.mean(nme) - def update(self, labels, preds): - self.count+=1 - label = labels[0].asnumpy() - pred_label = preds[-1].asnumpy() - nme = self.cal_nme(label, pred_label) + def update(self, labels, preds): + self.count += 1 + label = labels[0].asnumpy() + pred_label = preds[-1].asnumpy() + nme = self.cal_nme(label, pred_label) - #print('nme', nme) - #nme = np.mean(nme) - self.sum_metric += np.mean(nme) - self.num_inst += 1.0 + #print('nme', nme) + #nme = np.mean(nme) + self.sum_metric += np.mean(nme) + self.num_inst += 1.0 diff --git a/alignment/heatmapReg/optimizer.py b/alignment/heatmapReg/optimizer.py index d86970c..36290de 100644 --- a/alignment/heatmapReg/optimizer.py +++ b/alignment/heatmapReg/optimizer.py @@ -1,12 +1,19 @@ import mxnet as mx import mxnet.optimizer as optimizer -from mxnet.ndarray import (NDArray, zeros, clip, sqrt, cast, maximum, abs as NDabs) +from mxnet.ndarray import (NDArray, zeros, clip, sqrt, cast, maximum, abs as + NDabs) #from mxnet.ndarray import (sgd_update, sgd_mom_update, adam_update, rmsprop_update, rmspropalex_update, # mp_sgd_update, mp_sgd_mom_update, square, ftrl_update) + class ONadam(optimizer.Optimizer): - def __init__(self, learning_rate=0.001, beta1=0.9, beta2=0.999, epsilon=1e-8, - schedule_decay=0.004, **kwargs): + def __init__(self, + learning_rate=0.001, + beta1=0.9, + beta2=0.999, + epsilon=1e-8, + schedule_decay=0.004, + **kwargs): super(ONadam, self).__init__(learning_rate=learning_rate, **kwargs) self.beta1 = beta1 self.beta2 = beta2 @@ -15,12 +22,14 @@ class ONadam(optimizer.Optimizer): self.m_schedule = 1. def create_state(self, index, weight): - return (zeros(weight.shape, weight.context, dtype=weight.dtype), # mean - zeros(weight.shape, weight.context, dtype=weight.dtype)) # variance + return ( + zeros(weight.shape, weight.context, dtype=weight.dtype), # mean + zeros(weight.shape, weight.context, + dtype=weight.dtype)) # variance def update(self, index, weight, grad, state): - assert(isinstance(weight, NDArray)) - assert(isinstance(grad, NDArray)) + assert (isinstance(weight, NDArray)) + assert (isinstance(grad, NDArray)) self._update_count(index) lr = self._get_lr(index) wd = self._get_wd(index) @@ -34,8 +43,11 @@ class ONadam(optimizer.Optimizer): grad = clip(grad, -self.clip_gradient, self.clip_gradient) # warming momentum schedule - momentum_t = self.beta1 * (1. - 0.5 * (pow(0.96, t * self.schedule_decay))) - momentum_t_1 = self.beta1 * (1. - 0.5 * (pow(0.96, (t + 1) * self.schedule_decay))) + momentum_t = self.beta1 * (1. - 0.5 * + (pow(0.96, t * self.schedule_decay))) + momentum_t_1 = self.beta1 * (1. - 0.5 * + (pow(0.96, + (t + 1) * self.schedule_decay))) self.m_schedule = self.m_schedule * momentum_t m_schedule_next = self.m_schedule * momentum_t_1 @@ -51,4 +63,3 @@ class ONadam(optimizer.Optimizer): # update weight weight[:] -= lr * m_t_bar / (sqrt(v_t_prime) + self.epsilon) - diff --git a/alignment/heatmapReg/sample_config.py b/alignment/heatmapReg/sample_config.py index 0870638..bb9ada1 100644 --- a/alignment/heatmapReg/sample_config.py +++ b/alignment/heatmapReg/sample_config.py @@ -39,7 +39,6 @@ network.sdu.net_block = 'cab' network.sdu.net_binarize = False network.sdu.losstype = 'heatmap' - # dataset settings dataset = edict() @@ -67,7 +66,6 @@ dataset.i3d.output_label_size = 64 dataset.i3d.label_xfirst = False dataset.i3d.val_targets = ['AFLW2000-3D'] - # default settings default = edict() @@ -88,13 +86,13 @@ default.wd = 0.0 default.per_batch_size = 20 default.lr_step = '16000,24000,30000' + def generate_config(_network, _dataset): for k, v in network[_network].items(): - config[k] = v - default[k] = v + config[k] = v + default[k] = v for k, v in dataset[_dataset].items(): - config[k] = v - default[k] = v + config[k] = v + default[k] = v config.network = _network config.dataset = _dataset - diff --git a/alignment/heatmapReg/symbol/sym_heatmap.py b/alignment/heatmapReg/symbol/sym_heatmap.py index 12a7e94..3384112 100644 --- a/alignment/heatmapReg/symbol/sym_heatmap.py +++ b/alignment/heatmapReg/symbol/sym_heatmap.py @@ -5,25 +5,25 @@ import mxnet as mx import numpy as np from config import config - ACT_BIT = 1 bn_mom = 0.9 workspace = 256 memonger = False - def Conv(**kwargs): body = mx.sym.Convolution(**kwargs) return body + def Act(data, act_type, name): - if act_type=='prelu': - body = mx.sym.LeakyReLU(data = data, act_type='prelu', name = name) + if act_type == 'prelu': + body = mx.sym.LeakyReLU(data=data, act_type='prelu', name=name) else: - body = mx.symbol.Activation(data=data, act_type=act_type, name=name) + body = mx.symbol.Activation(data=data, act_type=act_type, name=name) return body + #def lin(data, num_filter, workspace, name, binarize, dcn): # bit = 1 # if not binarize: @@ -51,38 +51,92 @@ def Act(data, act_type, name): # conv1 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_bn2') # return conv1 + def lin3(data, num_filter, workspace, name, k, g=1, d=1): - if k!=3: - conv1 = Conv(data=data, num_filter=num_filter, kernel=(k,k), stride=(1,1), pad=((k-1)//2,(k-1)//2), num_group=g, - no_bias=True, workspace=workspace, name=name + '_conv') + if k != 3: + conv1 = Conv(data=data, + num_filter=num_filter, + kernel=(k, k), + stride=(1, 1), + pad=((k - 1) // 2, (k - 1) // 2), + num_group=g, + no_bias=True, + workspace=workspace, + name=name + '_conv') else: - conv1 = Conv(data=data, num_filter=num_filter, kernel=(k,k), stride=(1,1), pad=(d,d), num_group=g, dilate=(d, d), - no_bias=True, workspace=workspace, name=name + '_conv') - bn1 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_bn') + conv1 = Conv(data=data, + num_filter=num_filter, + kernel=(k, k), + stride=(1, 1), + pad=(d, d), + num_group=g, + dilate=(d, d), + no_bias=True, + workspace=workspace, + name=name + '_conv') + bn1 = mx.sym.BatchNorm(data=conv1, + fix_gamma=False, + momentum=bn_mom, + eps=2e-5, + name=name + '_bn') act1 = Act(data=bn1, act_type='relu', name=name + '_relu') ret = act1 return ret -def ConvFactory(data, num_filter, kernel, stride=(1, 1), pad=(0, 0), act_type="relu", mirror_attr={}, with_act=True, dcn=False, name=''): + +def ConvFactory(data, + num_filter, + kernel, + stride=(1, 1), + pad=(0, 0), + act_type="relu", + mirror_attr={}, + with_act=True, + dcn=False, + name=''): if not dcn: - conv = mx.symbol.Convolution( - data=data, num_filter=num_filter, kernel=kernel, stride=stride, pad=pad, no_bias=True, workspace=workspace, name=name+'_conv') + conv = mx.symbol.Convolution(data=data, + num_filter=num_filter, + kernel=kernel, + stride=stride, + pad=pad, + no_bias=True, + workspace=workspace, + name=name + '_conv') else: - conv_offset = mx.symbol.Convolution(name=name+'_conv_offset', data = data, - num_filter=18, pad=(1, 1), kernel=(3, 3), stride=(1, 1)) - conv = mx.contrib.symbol.DeformableConvolution(name=name+"_conv", data=data, offset=conv_offset, - num_filter=num_filter, pad=(1,1), kernel=(3,3), num_deformable_group=1, stride=stride, dilate=(1, 1), no_bias=False) - bn = mx.symbol.BatchNorm(data=conv, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name+'_bn') + conv_offset = mx.symbol.Convolution(name=name + '_conv_offset', + data=data, + num_filter=18, + pad=(1, 1), + kernel=(3, 3), + stride=(1, 1)) + conv = mx.contrib.symbol.DeformableConvolution(name=name + "_conv", + data=data, + offset=conv_offset, + num_filter=num_filter, + pad=(1, 1), + kernel=(3, 3), + num_deformable_group=1, + stride=stride, + dilate=(1, 1), + no_bias=False) + bn = mx.symbol.BatchNorm(data=conv, + fix_gamma=False, + momentum=bn_mom, + eps=2e-5, + name=name + '_bn') if with_act: - act = Act(bn, act_type, name=name+'_relu') - #act = mx.symbol.Activation( - # data=bn, act_type=act_type, attr=mirror_attr, name=name+'_relu') - return act + act = Act(bn, act_type, name=name + '_relu') + #act = mx.symbol.Activation( + # data=bn, act_type=act_type, attr=mirror_attr, name=name+'_relu') + return act else: - return bn + return bn + class CAB: - def __init__(self, data, nFilters, nModules, n, workspace, name, dilate, group): + def __init__(self, data, nFilters, nModules, n, workspace, name, dilate, + group): self.data = data self.nFilters = nFilters self.nModules = nModules @@ -98,34 +152,42 @@ class CAB: if key in self.sym_map: return self.sym_map[key] ret = None - if h==self.n: - if w==self.n: + if h == self.n: + if w == self.n: ret = (self.data, self.nFilters) else: - x = self.get_output(w+1, h) - f = int(x[1]*0.5) - if w!=self.n-1: - body = lin3(x[0], f, self.workspace, "%s_w%d_h%d_1"%(self.name, w, h), 3, self.group, 1) + x = self.get_output(w + 1, h) + f = int(x[1] * 0.5) + if w != self.n - 1: + body = lin3(x[0], f, self.workspace, + "%s_w%d_h%d_1" % (self.name, w, h), 3, + self.group, 1) else: - body = lin3(x[0], f, self.workspace, "%s_w%d_h%d_1"%(self.name, w, h), 3, self.group, self.dilate) - ret = (body,f) + body = lin3(x[0], f, self.workspace, + "%s_w%d_h%d_1" % (self.name, w, h), 3, + self.group, self.dilate) + ret = (body, f) else: - x = self.get_output(w+1, h+1) - y = self.get_output(w, h+1) - if h%2==1 and h!=w: - xbody = lin3(x[0], x[1], self.workspace, "%s_w%d_h%d_2"%(self.name, w, h), 3, x[1]) + x = self.get_output(w + 1, h + 1) + y = self.get_output(w, h + 1) + if h % 2 == 1 and h != w: + xbody = lin3(x[0], x[1], self.workspace, + "%s_w%d_h%d_2" % (self.name, w, h), 3, x[1]) #xbody = xbody+x[0] else: xbody = x[0] #xbody = x[0] #xbody = lin3(x[0], x[1], self.workspace, "%s_w%d_h%d_2"%(self.name, w, h), 3, x[1]) - if w==0: - ybody = lin3(y[0], y[1], self.workspace, "%s_w%d_h%d_3"%(self.name, w, h), 3, self.group) + if w == 0: + ybody = lin3(y[0], y[1], self.workspace, + "%s_w%d_h%d_3" % (self.name, w, h), 3, self.group) else: ybody = y[0] ybody = mx.sym.concat(y[0], ybody, dim=1) - body = mx.sym.add_n(xbody,ybody, name="%s_w%d_h%d_add"%(self.name, w, h)) - body = body/2 + body = mx.sym.add_n(xbody, + ybody, + name="%s_w%d_h%d_add" % (self.name, w, h)) + body = body / 2 ret = (body, x[1]) self.sym_map[key] = ret return ret @@ -133,116 +195,321 @@ class CAB: def get(self): return self.get_output(1, 1)[0] -def conv_resnet(data, num_filter, stride, dim_match, name, binarize, dcn, dilate, **kwargs): + +def conv_resnet(data, num_filter, stride, dim_match, name, binarize, dcn, + dilate, **kwargs): bit = 1 #print('in unit2') # the same as https://github.com/facebook/fb.resnet.torch#notes, a bit difference with origin paper - bn1 = mx.sym.BatchNorm(data=data, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn1') + bn1 = mx.sym.BatchNorm(data=data, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn1') if not binarize: - act1 = Act(data=bn1, act_type='relu', name=name + '_relu1') - conv1 = Conv(data=act1, num_filter=int(num_filter*0.5), kernel=(1,1), stride=(1,1), pad=(0,0), - no_bias=True, workspace=workspace, name=name + '_conv1') + act1 = Act(data=bn1, act_type='relu', name=name + '_relu1') + conv1 = Conv(data=act1, + num_filter=int(num_filter * 0.5), + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + no_bias=True, + workspace=workspace, + name=name + '_conv1') else: - act1 = mx.sym.QActivation(data=bn1, act_bit=ACT_BIT, name=name + '_relu1', backward_only=True) - conv1 = mx.sym.QConvolution(data=act1, num_filter=int(num_filter*0.5), kernel=(1,1), stride=(1,1), pad=(0,0), - no_bias=True, workspace=workspace, name=name + '_conv1', act_bit=ACT_BIT, weight_bit=bit) - bn2 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn2') + act1 = mx.sym.QActivation(data=bn1, + act_bit=ACT_BIT, + name=name + '_relu1', + backward_only=True) + conv1 = mx.sym.QConvolution(data=act1, + num_filter=int(num_filter * 0.5), + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + no_bias=True, + workspace=workspace, + name=name + '_conv1', + act_bit=ACT_BIT, + weight_bit=bit) + bn2 = mx.sym.BatchNorm(data=conv1, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn2') if not binarize: - act2 = Act(data=bn2, act_type='relu', name=name + '_relu2') - conv2 = Conv(data=act2, num_filter=int(num_filter*0.5), kernel=(3,3), stride=(1,1), pad=(1,1), - no_bias=True, workspace=workspace, name=name + '_conv2') + act2 = Act(data=bn2, act_type='relu', name=name + '_relu2') + conv2 = Conv(data=act2, + num_filter=int(num_filter * 0.5), + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv2') else: - act2 = mx.sym.QActivation(data=bn2, act_bit=ACT_BIT, name=name + '_relu2', backward_only=True) - conv2 = mx.sym.QConvolution(data=act2, num_filter=int(num_filter*0.5), kernel=(3,3), stride=(1,1), pad=(1,1), - no_bias=True, workspace=workspace, name=name + '_conv2', act_bit=ACT_BIT, weight_bit=bit) - bn3 = mx.sym.BatchNorm(data=conv2, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn3') + act2 = mx.sym.QActivation(data=bn2, + act_bit=ACT_BIT, + name=name + '_relu2', + backward_only=True) + conv2 = mx.sym.QConvolution(data=act2, + num_filter=int(num_filter * 0.5), + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv2', + act_bit=ACT_BIT, + weight_bit=bit) + bn3 = mx.sym.BatchNorm(data=conv2, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn3') if not binarize: - act3 = Act(data=bn3, act_type='relu', name=name + '_relu3') - conv3 = Conv(data=act3, num_filter=num_filter, kernel=(1,1), stride=(1,1), pad=(0,0), no_bias=True, - workspace=workspace, name=name + '_conv3') + act3 = Act(data=bn3, act_type='relu', name=name + '_relu3') + conv3 = Conv(data=act3, + num_filter=num_filter, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + no_bias=True, + workspace=workspace, + name=name + '_conv3') else: - act3 = mx.sym.QActivation(data=bn3, act_bit=ACT_BIT, name=name + '_relu3', backward_only=True) - conv3 = mx.sym.QConvolution(data=act3, num_filter=num_filter, kernel=(1,1), stride=(1,1), pad=(0,0), - no_bias=True, workspace=workspace, name=name + '_conv3', act_bit=ACT_BIT, weight_bit=bit) + act3 = mx.sym.QActivation(data=bn3, + act_bit=ACT_BIT, + name=name + '_relu3', + backward_only=True) + conv3 = mx.sym.QConvolution(data=act3, + num_filter=num_filter, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + no_bias=True, + workspace=workspace, + name=name + '_conv3', + act_bit=ACT_BIT, + weight_bit=bit) #if binarize: # conv3 = mx.sym.BatchNorm(data=conv3, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn4') if dim_match: shortcut = data else: if not binarize: - shortcut = Conv(data=act1, num_filter=num_filter, kernel=(1,1), stride=stride, no_bias=True, - workspace=workspace, name=name+'_sc') + shortcut = Conv(data=act1, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + no_bias=True, + workspace=workspace, + name=name + '_sc') else: - shortcut = mx.sym.QConvolution(data=act1, num_filter=num_filter, kernel=(1,1), stride=stride, pad=(0,0), - no_bias=True, workspace=workspace, name=name + '_sc', act_bit=ACT_BIT, weight_bit=bit) + shortcut = mx.sym.QConvolution(data=act1, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + pad=(0, 0), + no_bias=True, + workspace=workspace, + name=name + '_sc', + act_bit=ACT_BIT, + weight_bit=bit) if memonger: shortcut._set_attr(mirror_stage='True') return conv3 + shortcut -def conv_hpm(data, num_filter, stride, dim_match, name, binarize, dcn, dilation, **kwargs): +def conv_hpm(data, num_filter, stride, dim_match, name, binarize, dcn, + dilation, **kwargs): bit = 1 #print('in unit2') # the same as https://github.com/facebook/fb.resnet.torch#notes, a bit difference with origin paper - bn1 = mx.sym.BatchNorm(data=data, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn1') + bn1 = mx.sym.BatchNorm(data=data, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn1') if not binarize: - act1 = Act(data=bn1, act_type='relu', name=name + '_relu1') - if not dcn: - conv1 = Conv(data=act1, num_filter=int(num_filter*0.5), kernel=(3,3), stride=(1,1), pad=(dilation,dilation), dilate=(dilation,dilation), - no_bias=True, workspace=workspace, name=name + '_conv1') - else: - conv1_offset = mx.symbol.Convolution(name=name+'_conv1_offset', data = act1, - num_filter=18, pad=(1, 1), kernel=(3, 3), stride=(1, 1)) - conv1 = mx.contrib.symbol.DeformableConvolution(name=name+'_conv1', data=act1, offset=conv1_offset, - num_filter=int(num_filter*0.5), pad=(1,1), kernel=(3, 3), num_deformable_group=1, stride=(1, 1), dilate=(1, 1), no_bias=True) + act1 = Act(data=bn1, act_type='relu', name=name + '_relu1') + if not dcn: + conv1 = Conv(data=act1, + num_filter=int(num_filter * 0.5), + kernel=(3, 3), + stride=(1, 1), + pad=(dilation, dilation), + dilate=(dilation, dilation), + no_bias=True, + workspace=workspace, + name=name + '_conv1') + else: + conv1_offset = mx.symbol.Convolution(name=name + '_conv1_offset', + data=act1, + num_filter=18, + pad=(1, 1), + kernel=(3, 3), + stride=(1, 1)) + conv1 = mx.contrib.symbol.DeformableConvolution( + name=name + '_conv1', + data=act1, + offset=conv1_offset, + num_filter=int(num_filter * 0.5), + pad=(1, 1), + kernel=(3, 3), + num_deformable_group=1, + stride=(1, 1), + dilate=(1, 1), + no_bias=True) else: - act1 = mx.sym.QActivation(data=bn1, act_bit=ACT_BIT, name=name + '_relu1', backward_only=True) - conv1 = mx.sym.QConvolution_v1(data=act1, num_filter=int(num_filter*0.5), kernel=(3,3), stride=(1,1), pad=(1,1), - no_bias=True, workspace=workspace, name=name + '_conv1', act_bit=ACT_BIT, weight_bit=bit) - bn2 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn2') + act1 = mx.sym.QActivation(data=bn1, + act_bit=ACT_BIT, + name=name + '_relu1', + backward_only=True) + conv1 = mx.sym.QConvolution_v1(data=act1, + num_filter=int(num_filter * 0.5), + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv1', + act_bit=ACT_BIT, + weight_bit=bit) + bn2 = mx.sym.BatchNorm(data=conv1, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn2') if not binarize: - act2 = Act(data=bn2, act_type='relu', name=name + '_relu2') - if not dcn: - conv2 = Conv(data=act2, num_filter=int(num_filter*0.25), kernel=(3,3), stride=(1,1), pad=(dilation,dilation), dilate=(dilation,dilation), - no_bias=True, workspace=workspace, name=name + '_conv2') - else: - conv2_offset = mx.symbol.Convolution(name=name+'_conv2_offset', data = act2, - num_filter=18, pad=(1, 1), kernel=(3, 3), stride=(1, 1)) - conv2 = mx.contrib.symbol.DeformableConvolution(name=name+'_conv2', data=act2, offset=conv2_offset, - num_filter=int(num_filter*0.25), pad=(1,1), kernel=(3, 3), num_deformable_group=1, stride=(1, 1), dilate=(1, 1), no_bias=True) + act2 = Act(data=bn2, act_type='relu', name=name + '_relu2') + if not dcn: + conv2 = Conv(data=act2, + num_filter=int(num_filter * 0.25), + kernel=(3, 3), + stride=(1, 1), + pad=(dilation, dilation), + dilate=(dilation, dilation), + no_bias=True, + workspace=workspace, + name=name + '_conv2') + else: + conv2_offset = mx.symbol.Convolution(name=name + '_conv2_offset', + data=act2, + num_filter=18, + pad=(1, 1), + kernel=(3, 3), + stride=(1, 1)) + conv2 = mx.contrib.symbol.DeformableConvolution( + name=name + '_conv2', + data=act2, + offset=conv2_offset, + num_filter=int(num_filter * 0.25), + pad=(1, 1), + kernel=(3, 3), + num_deformable_group=1, + stride=(1, 1), + dilate=(1, 1), + no_bias=True) else: - act2 = mx.sym.QActivation(data=bn2, act_bit=ACT_BIT, name=name + '_relu2', backward_only=True) - conv2 = mx.sym.QConvolution_v1(data=act2, num_filter=int(num_filter*0.25), kernel=(3,3), stride=(1,1), pad=(1,1), - no_bias=True, workspace=workspace, name=name + '_conv2', act_bit=ACT_BIT, weight_bit=bit) - bn3 = mx.sym.BatchNorm(data=conv2, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn3') + act2 = mx.sym.QActivation(data=bn2, + act_bit=ACT_BIT, + name=name + '_relu2', + backward_only=True) + conv2 = mx.sym.QConvolution_v1(data=act2, + num_filter=int(num_filter * 0.25), + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv2', + act_bit=ACT_BIT, + weight_bit=bit) + bn3 = mx.sym.BatchNorm(data=conv2, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn3') if not binarize: - act3 = Act(data=bn3, act_type='relu', name=name + '_relu3') - if not dcn: - conv3 = Conv(data=act3, num_filter=int(num_filter*0.25), kernel=(3,3), stride=(1,1), pad=(dilation,dilation), dilate=(dilation,dilation), - no_bias=True, workspace=workspace, name=name + '_conv3') - else: - conv3_offset = mx.symbol.Convolution(name=name+'_conv3_offset', data = act3, - num_filter=18, pad=(1, 1), kernel=(3, 3), stride=(1, 1)) - conv3 = mx.contrib.symbol.DeformableConvolution(name=name+'_conv3', data=act3, offset=conv3_offset, - num_filter=int(num_filter*0.25), pad=(1,1), kernel=(3, 3), num_deformable_group=1, stride=(1, 1), dilate=(1, 1), no_bias=True) + act3 = Act(data=bn3, act_type='relu', name=name + '_relu3') + if not dcn: + conv3 = Conv(data=act3, + num_filter=int(num_filter * 0.25), + kernel=(3, 3), + stride=(1, 1), + pad=(dilation, dilation), + dilate=(dilation, dilation), + no_bias=True, + workspace=workspace, + name=name + '_conv3') + else: + conv3_offset = mx.symbol.Convolution(name=name + '_conv3_offset', + data=act3, + num_filter=18, + pad=(1, 1), + kernel=(3, 3), + stride=(1, 1)) + conv3 = mx.contrib.symbol.DeformableConvolution( + name=name + '_conv3', + data=act3, + offset=conv3_offset, + num_filter=int(num_filter * 0.25), + pad=(1, 1), + kernel=(3, 3), + num_deformable_group=1, + stride=(1, 1), + dilate=(1, 1), + no_bias=True) else: - act3 = mx.sym.QActivation(data=bn3, act_bit=ACT_BIT, name=name + '_relu3', backward_only=True) - conv3 = mx.sym.QConvolution_v1(data=act3, num_filter=int(num_filter*0.25), kernel=(3,3), stride=(1,1), pad=(1,1), - no_bias=True, workspace=workspace, name=name + '_conv3', act_bit=ACT_BIT, weight_bit=bit) + act3 = mx.sym.QActivation(data=bn3, + act_bit=ACT_BIT, + name=name + '_relu3', + backward_only=True) + conv3 = mx.sym.QConvolution_v1(data=act3, + num_filter=int(num_filter * 0.25), + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv3', + act_bit=ACT_BIT, + weight_bit=bit) conv4 = mx.symbol.Concat(*[conv1, conv2, conv3]) if binarize: - conv4 = mx.sym.BatchNorm(data=conv4, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn4') + conv4 = mx.sym.BatchNorm(data=conv4, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn4') if dim_match: shortcut = data else: if not binarize: - shortcut = Conv(data=act1, num_filter=num_filter, kernel=(1,1), stride=stride, no_bias=True, - workspace=workspace, name=name+'_sc') + shortcut = Conv(data=act1, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + no_bias=True, + workspace=workspace, + name=name + '_sc') else: - #assert(False) - shortcut = mx.sym.QConvolution_v1(data=act1, num_filter=num_filter, kernel=(1,1), stride=stride, pad=(0,0), - no_bias=True, workspace=workspace, name=name + '_sc', act_bit=ACT_BIT, weight_bit=bit) - shortcut = mx.sym.BatchNorm(data=shortcut, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_sc_bn') + #assert(False) + shortcut = mx.sym.QConvolution_v1(data=act1, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + pad=(0, 0), + no_bias=True, + workspace=workspace, + name=name + '_sc', + act_bit=ACT_BIT, + weight_bit=bit) + shortcut = mx.sym.BatchNorm(data=shortcut, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_sc_bn') if memonger: shortcut._set_attr(mirror_stage='True') return conv4 + shortcut @@ -250,85 +517,153 @@ def conv_hpm(data, num_filter, stride, dim_match, name, binarize, dcn, dilation, #return act4 + shortcut -def block17(net, input_num_channels, scale=1.0, with_act=True, act_type='relu', mirror_attr={}, name=''): - tower_conv = ConvFactory(net, 192, (1, 1), name=name+'_conv') - tower_conv1_0 = ConvFactory(net, 129, (1, 1), name=name+'_conv1_0') - tower_conv1_1 = ConvFactory(tower_conv1_0, 160, (1, 7), pad=(1, 2), name=name+'_conv1_1') - tower_conv1_2 = ConvFactory(tower_conv1_1, 192, (7, 1), pad=(2, 1), name=name+'_conv1_2') +def block17(net, + input_num_channels, + scale=1.0, + with_act=True, + act_type='relu', + mirror_attr={}, + name=''): + tower_conv = ConvFactory(net, 192, (1, 1), name=name + '_conv') + tower_conv1_0 = ConvFactory(net, 129, (1, 1), name=name + '_conv1_0') + tower_conv1_1 = ConvFactory(tower_conv1_0, + 160, (1, 7), + pad=(1, 2), + name=name + '_conv1_1') + tower_conv1_2 = ConvFactory(tower_conv1_1, + 192, (7, 1), + pad=(2, 1), + name=name + '_conv1_2') tower_mixed = mx.symbol.Concat(*[tower_conv, tower_conv1_2]) - tower_out = ConvFactory( - tower_mixed, input_num_channels, (1, 1), with_act=False, name=name+'_conv_out') - net = net+scale * tower_out + tower_out = ConvFactory(tower_mixed, + input_num_channels, (1, 1), + with_act=False, + name=name + '_conv_out') + net = net + scale * tower_out if with_act: - act = mx.symbol.Activation( - data=net, act_type=act_type, attr=mirror_attr) + act = mx.symbol.Activation(data=net, + act_type=act_type, + attr=mirror_attr) return act else: return net -def block35(net, input_num_channels, scale=1.0, with_act=True, act_type='relu', mirror_attr={}, name=''): + +def block35(net, + input_num_channels, + scale=1.0, + with_act=True, + act_type='relu', + mirror_attr={}, + name=''): M = 1.0 - tower_conv = ConvFactory(net, int(input_num_channels*0.25*M), (1, 1), name=name+'_conv') - tower_conv1_0 = ConvFactory(net, int(input_num_channels*0.25*M), (1, 1), name=name+'_conv1_0') - tower_conv1_1 = ConvFactory(tower_conv1_0, int(input_num_channels*0.25*M), (3, 3), pad=(1, 1), name=name+'_conv1_1') - tower_conv2_0 = ConvFactory(net, int(input_num_channels*0.25*M), (1, 1), name=name+'_conv2_0') - tower_conv2_1 = ConvFactory(tower_conv2_0, int(input_num_channels*0.375*M), (3, 3), pad=(1, 1), name=name+'_conv2_1') - tower_conv2_2 = ConvFactory(tower_conv2_1, int(input_num_channels*0.5*M), (3, 3), pad=(1, 1), name=name+'_conv2_2') + tower_conv = ConvFactory(net, + int(input_num_channels * 0.25 * M), (1, 1), + name=name + '_conv') + tower_conv1_0 = ConvFactory(net, + int(input_num_channels * 0.25 * M), (1, 1), + name=name + '_conv1_0') + tower_conv1_1 = ConvFactory(tower_conv1_0, + int(input_num_channels * 0.25 * M), (3, 3), + pad=(1, 1), + name=name + '_conv1_1') + tower_conv2_0 = ConvFactory(net, + int(input_num_channels * 0.25 * M), (1, 1), + name=name + '_conv2_0') + tower_conv2_1 = ConvFactory(tower_conv2_0, + int(input_num_channels * 0.375 * M), (3, 3), + pad=(1, 1), + name=name + '_conv2_1') + tower_conv2_2 = ConvFactory(tower_conv2_1, + int(input_num_channels * 0.5 * M), (3, 3), + pad=(1, 1), + name=name + '_conv2_2') tower_mixed = mx.symbol.Concat(*[tower_conv, tower_conv1_1, tower_conv2_2]) - tower_out = ConvFactory( - tower_mixed, input_num_channels, (1, 1), with_act=False, name=name+'_conv_out') + tower_out = ConvFactory(tower_mixed, + input_num_channels, (1, 1), + with_act=False, + name=name + '_conv_out') - net = net+scale * tower_out + net = net + scale * tower_out if with_act: - act = mx.symbol.Activation( - data=net, act_type=act_type, attr=mirror_attr) + act = mx.symbol.Activation(data=net, + act_type=act_type, + attr=mirror_attr) return act else: return net -def conv_inception(data, num_filter, stride, dim_match, name, binarize, dcn, dilate, **kwargs): + +def conv_inception(data, num_filter, stride, dim_match, name, binarize, dcn, + dilate, **kwargs): assert not binarize - if stride[0]>1 or not dim_match: - return conv_resnet(data, num_filter, stride, dim_match, name, binarize, dcn, dilate, **kwargs) - conv4 = block35(data, num_filter, name=name+'_block35') + if stride[0] > 1 or not dim_match: + return conv_resnet(data, num_filter, stride, dim_match, name, binarize, + dcn, dilate, **kwargs) + conv4 = block35(data, num_filter, name=name + '_block35') return conv4 -def conv_cab(data, num_filter, stride, dim_match, name, binarize, dcn, dilate, **kwargs): - if stride[0]>1 or not dim_match: - return conv_hpm(data, num_filter, stride, dim_match, name, binarize, dcn, dilate, **kwargs) + +def conv_cab(data, num_filter, stride, dim_match, name, binarize, dcn, dilate, + **kwargs): + if stride[0] > 1 or not dim_match: + return conv_hpm(data, num_filter, stride, dim_match, name, binarize, + dcn, dilate, **kwargs) cab = CAB(data, num_filter, 1, 4, workspace, name, dilate, 1) return cab.get() -def conv_block(data, num_filter, stride, dim_match, name, binarize, dcn, dilate): - if config.net_block=='resnet': - return conv_resnet(data, num_filter, stride, dim_match, name, binarize, dcn, dilate) - elif config.net_block=='inception': - return conv_inception(data, num_filter, stride, dim_match, name, binarize, dcn, dilate) - elif config.net_block=='hpm': - return conv_hpm(data, num_filter, stride, dim_match, name, binarize, dcn, dilate) - elif config.net_block=='cab': - return conv_cab(data, num_filter, stride, dim_match, name, binarize, dcn, dilate) + +def conv_block(data, num_filter, stride, dim_match, name, binarize, dcn, + dilate): + if config.net_block == 'resnet': + return conv_resnet(data, num_filter, stride, dim_match, name, binarize, + dcn, dilate) + elif config.net_block == 'inception': + return conv_inception(data, num_filter, stride, dim_match, name, + binarize, dcn, dilate) + elif config.net_block == 'hpm': + return conv_hpm(data, num_filter, stride, dim_match, name, binarize, + dcn, dilate) + elif config.net_block == 'cab': + return conv_cab(data, num_filter, stride, dim_match, name, binarize, + dcn, dilate) + def hourglass(data, nFilters, nModules, n, workspace, name, binarize, dcn): - s = 2 - _dcn = False - up1 = data - for i in range(nModules): - up1 = conv_block(up1, nFilters, (1,1), True, "%s_up1_%d"%(name,i), binarize, _dcn, 1) - low1 = mx.sym.Pooling(data=data, kernel=(s, s), stride=(s,s), pad=(0,0), pool_type='max') - for i in range(nModules): - low1 = conv_block(low1, nFilters, (1,1), True, "%s_low1_%d"%(name,i), binarize, _dcn, 1) - if n>1: - low2 = hourglass(low1, nFilters, nModules, n-1, workspace, "%s_%d"%(name, n-1), binarize, dcn) - else: - low2 = low1 + s = 2 + _dcn = False + up1 = data for i in range(nModules): - low2 = conv_block(low2, nFilters, (1,1), True, "%s_low2_%d"%(name,i), binarize, _dcn, 1) #TODO - low3 = low2 - for i in range(nModules): - low3 = conv_block(low3, nFilters, (1,1), True, "%s_low3_%d"%(name,i), binarize, _dcn, 1) - up2 = mx.symbol.UpSampling(low3, scale=s, sample_type='nearest', workspace=512, name='%s_upsampling_%s'%(name,n), num_args=1) - return mx.symbol.add_n(up1, up2) + up1 = conv_block(up1, nFilters, (1, 1), True, "%s_up1_%d" % (name, i), + binarize, _dcn, 1) + low1 = mx.sym.Pooling(data=data, + kernel=(s, s), + stride=(s, s), + pad=(0, 0), + pool_type='max') + for i in range(nModules): + low1 = conv_block(low1, nFilters, (1, 1), True, + "%s_low1_%d" % (name, i), binarize, _dcn, 1) + if n > 1: + low2 = hourglass(low1, nFilters, nModules, n - 1, workspace, + "%s_%d" % (name, n - 1), binarize, dcn) + else: + low2 = low1 + for i in range(nModules): + low2 = conv_block(low2, nFilters, (1, 1), True, + "%s_low2_%d" % (name, i), binarize, _dcn, + 1) #TODO + low3 = low2 + for i in range(nModules): + low3 = conv_block(low3, nFilters, (1, 1), True, + "%s_low3_%d" % (name, i), binarize, _dcn, 1) + up2 = mx.symbol.UpSampling(low3, + scale=s, + sample_type='nearest', + workspace=512, + name='%s_upsampling_%s' % (name, n), + num_args=1) + return mx.symbol.add_n(up1, up2) class STA: @@ -341,78 +676,133 @@ class STA: self.name = name self.sym_map = {} - def get_conv(self, data, name, dilate=1, group=1): - cab = CAB(data, self.nFilters, self.nModules, 4, self.workspace, name, dilate, group) + cab = CAB(data, self.nFilters, self.nModules, 4, self.workspace, name, + dilate, group) return cab.get() def get_output(self, w, h): #print(w,h) - assert w>=1 and w<=config.net_n+1 - assert h>=1 and h<=config.net_n+1 + assert w >= 1 and w <= config.net_n + 1 + assert h >= 1 and h <= config.net_n + 1 s = 2 bn_mom = 0.9 - key = (w,h) + key = (w, h) if key in self.sym_map: return self.sym_map[key] ret = None - if h==self.n: - if w==self.n: - ret = self.data,64 + if h == self.n: + if w == self.n: + ret = self.data, 64 else: - x = self.get_output(w+1, h) - body = self.get_conv(x[0], "%s_w%d_h%d_1"%(self.name, w, h)) - body = mx.sym.Pooling(data=body, kernel=(s, s), stride=(s,s), pad=(0,0), pool_type='max') - body = self.get_conv(body, "%s_w%d_h%d_2"%(self.name, w, h)) - ret = body, x[1]//2 + x = self.get_output(w + 1, h) + body = self.get_conv(x[0], "%s_w%d_h%d_1" % (self.name, w, h)) + body = mx.sym.Pooling(data=body, + kernel=(s, s), + stride=(s, s), + pad=(0, 0), + pool_type='max') + body = self.get_conv(body, "%s_w%d_h%d_2" % (self.name, w, h)) + ret = body, x[1] // 2 else: - x = self.get_output(w+1, h+1) - y = self.get_output(w, h+1) + x = self.get_output(w + 1, h + 1) + y = self.get_output(w, h + 1) HC = False - if h%2==1 and h!=w: - xbody = lin3(x[0], self.nFilters, self.workspace, "%s_w%d_h%d_x"%(self.name, w, h), 3, self.nFilters, 1) + if h % 2 == 1 and h != w: + xbody = lin3(x[0], self.nFilters, self.workspace, + "%s_w%d_h%d_x" % (self.name, w, h), 3, + self.nFilters, 1) HC = True #xbody = x[0] else: xbody = x[0] - if x[1]//y[1]==2: - if w>1: - ybody = mx.symbol.Deconvolution(data=y[0], num_filter=self.nFilters, kernel=(s,s), - stride=(s, s), - name='%s_upsampling_w%d_h%d'%(self.name,w, h), - attr={'lr_mult': '1.0'}, workspace=self.workspace) - ybody = mx.sym.BatchNorm(data=ybody, fix_gamma=False, momentum=bn_mom, eps=2e-5, name="%s_w%d_h%d_y_bn"%(self.name, w, h)) - ybody = Act(data=ybody, act_type='relu', name="%s_w%d_h%d_y_act"%(self.name, w, h)) + if x[1] // y[1] == 2: + if w > 1: + ybody = mx.symbol.Deconvolution( + data=y[0], + num_filter=self.nFilters, + kernel=(s, s), + stride=(s, s), + name='%s_upsampling_w%d_h%d' % (self.name, w, h), + attr={'lr_mult': '1.0'}, + workspace=self.workspace) + ybody = mx.sym.BatchNorm(data=ybody, + fix_gamma=False, + momentum=bn_mom, + eps=2e-5, + name="%s_w%d_h%d_y_bn" % + (self.name, w, h)) + ybody = Act(data=ybody, + act_type='relu', + name="%s_w%d_h%d_y_act" % (self.name, w, h)) else: - if h>=1: - ybody = mx.symbol.UpSampling(y[0], scale=s, sample_type='nearest', workspace=512, name='%s_upsampling_w%d_h%d'%(self.name,w, h), num_args=1) - ybody = self.get_conv(ybody, "%s_w%d_h%d_4"%(self.name, w, h)) + if h >= 1: + ybody = mx.symbol.UpSampling( + y[0], + scale=s, + sample_type='nearest', + workspace=512, + name='%s_upsampling_w%d_h%d' % (self.name, w, h), + num_args=1) + ybody = self.get_conv( + ybody, "%s_w%d_h%d_4" % (self.name, w, h)) else: - ybody = mx.symbol.Deconvolution(data=y[0], num_filter=self.nFilters, kernel=(s,s), - stride=(s, s), - name='%s_upsampling_w%d_h%d'%(self.name,w, h), - attr={'lr_mult': '1.0'}, workspace=self.workspace) - ybody = mx.sym.BatchNorm(data=ybody, fix_gamma=False, momentum=bn_mom, eps=2e-5, name="%s_w%d_h%d_y_bn"%(self.name, w, h)) - ybody = Act(data=ybody, act_type='relu', name="%s_w%d_h%d_y_act"%(self.name, w, h)) - ybody = Conv(data=ybody, num_filter=self.nFilters, kernel=(3,3), stride=(1,1), pad=(1,1), - no_bias=True, name="%s_w%d_h%d_y_conv2"%(self.name, w, h), workspace=self.workspace) - ybody = mx.sym.BatchNorm(data=ybody, fix_gamma=False, momentum=bn_mom, eps=2e-5, name="%s_w%d_h%d_y_bn2"%(self.name, w, h)) - ybody = Act(data=ybody, act_type='relu', name="%s_w%d_h%d_y_act2"%(self.name, w, h)) + ybody = mx.symbol.Deconvolution( + data=y[0], + num_filter=self.nFilters, + kernel=(s, s), + stride=(s, s), + name='%s_upsampling_w%d_h%d' % (self.name, w, h), + attr={'lr_mult': '1.0'}, + workspace=self.workspace) + ybody = mx.sym.BatchNorm(data=ybody, + fix_gamma=False, + momentum=bn_mom, + eps=2e-5, + name="%s_w%d_h%d_y_bn" % + (self.name, w, h)) + ybody = Act(data=ybody, + act_type='relu', + name="%s_w%d_h%d_y_act" % + (self.name, w, h)) + ybody = Conv(data=ybody, + num_filter=self.nFilters, + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + name="%s_w%d_h%d_y_conv2" % + (self.name, w, h), + workspace=self.workspace) + ybody = mx.sym.BatchNorm(data=ybody, + fix_gamma=False, + momentum=bn_mom, + eps=2e-5, + name="%s_w%d_h%d_y_bn2" % + (self.name, w, h)) + ybody = Act(data=ybody, + act_type='relu', + name="%s_w%d_h%d_y_act2" % + (self.name, w, h)) else: - ybody = self.get_conv(y[0], "%s_w%d_h%d_5"%(self.name, w, h)) + ybody = self.get_conv(y[0], "%s_w%d_h%d_5" % (self.name, w, h)) #if not HC: - if config.net_sta==2 and h==3 and w==2: - z = self.get_output(w+1, h) - zbody = z[0] - zbody = mx.sym.Pooling(data=zbody, kernel=(z[1], z[1]), stride=(z[1],z[1]), pad=(0,0), pool_type='avg') - body = xbody+ybody - body = body/2 - body = mx.sym.broadcast_mul(body, zbody) - else: #sta==1 - body = xbody+ybody - body = body/2 + if config.net_sta == 2 and h == 3 and w == 2: + z = self.get_output(w + 1, h) + zbody = z[0] + zbody = mx.sym.Pooling(data=zbody, + kernel=(z[1], z[1]), + stride=(z[1], z[1]), + pad=(0, 0), + pool_type='avg') + body = xbody + ybody + body = body / 2 + body = mx.sym.broadcast_mul(body, zbody) + else: #sta==1 + body = xbody + ybody + body = body / 2 ret = body, x[1] assert ret is not None @@ -422,53 +812,59 @@ class STA: def get(self): return self.get_output(1, 1)[0] -class SymCoherent: - def __init__(self, per_batch_size): - self.per_batch_size = per_batch_size - self.flip_order = [16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, - 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 27, 28, 29, 30, 35, 34, 33, 32, 31, - 45, 44, 43, 42, 47, 46, 39, 38, 37, 36, 41, 40, 54, 53, 52, 51, 50, 49, 48, - 59, 58, 57, 56, 55, 64, 63, 62, 61, 60, 67, 66, 65] - def get(self, data): - #data.shape[0]==per_batch_size - b = self.per_batch_size//2 - ux = mx.sym.slice_axis(data, axis=0, begin=0, end=b) - dx = mx.sym.slice_axis(data, axis=0, begin=b, end=b*2) - ux = mx.sym.flip(ux, axis=3) - #ux = mx.sym.take(ux, indices = self.flip_order, axis=0) - ux_list = [] - for o in self.flip_order: - _ux = mx.sym.slice_axis(ux, axis=1, begin=o, end=o+1) - ux_list.append(_ux) - ux = mx.sym.concat(*ux_list, dim=1) - return ux, dx +class SymCoherent: + def __init__(self, per_batch_size): + self.per_batch_size = per_batch_size + self.flip_order = [ + 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 26, 25, + 24, 23, 22, 21, 20, 19, 18, 17, 27, 28, 29, 30, 35, 34, 33, 32, 31, + 45, 44, 43, 42, 47, 46, 39, 38, 37, 36, 41, 40, 54, 53, 52, 51, 50, + 49, 48, 59, 58, 57, 56, 55, 64, 63, 62, 61, 60, 67, 66, 65 + ] + + def get(self, data): + #data.shape[0]==per_batch_size + b = self.per_batch_size // 2 + ux = mx.sym.slice_axis(data, axis=0, begin=0, end=b) + dx = mx.sym.slice_axis(data, axis=0, begin=b, end=b * 2) + ux = mx.sym.flip(ux, axis=3) + #ux = mx.sym.take(ux, indices = self.flip_order, axis=0) + ux_list = [] + for o in self.flip_order: + _ux = mx.sym.slice_axis(ux, axis=1, begin=o, end=o + 1) + ux_list.append(_ux) + ux = mx.sym.concat(*ux_list, dim=1) + return ux, dx + def l2_loss(x, y): - loss = x-y - loss = mx.symbol.smooth_l1(loss, scalar=1.0) - #loss = loss*loss - loss = mx.symbol.mean(loss) - return loss + loss = x - y + loss = mx.symbol.smooth_l1(loss, scalar=1.0) + #loss = loss*loss + loss = mx.symbol.mean(loss) + return loss + def ce_loss(x, y): - #loss = mx.sym.SoftmaxOutput(data = x, label = y, normalization='valid', multi_output=True) - x_max = mx.sym.max(x, axis=[2,3], keepdims=True) - x = mx.sym.broadcast_minus(x, x_max) - body = mx.sym.exp(x) - sums = mx.sym.sum(body, axis=[2,3], keepdims=True) - body = mx.sym.broadcast_div(body, sums) - loss = mx.sym.log(body) - loss = loss*y*-1.0 - loss = mx.symbol.mean(loss, axis=[1,2,3]) - #loss = mx.symbol.mean(loss) - return loss + #loss = mx.sym.SoftmaxOutput(data = x, label = y, normalization='valid', multi_output=True) + x_max = mx.sym.max(x, axis=[2, 3], keepdims=True) + x = mx.sym.broadcast_minus(x, x_max) + body = mx.sym.exp(x) + sums = mx.sym.sum(body, axis=[2, 3], keepdims=True) + body = mx.sym.broadcast_div(body, sums) + loss = mx.sym.log(body) + loss = loss * y * -1.0 + loss = mx.symbol.mean(loss, axis=[1, 2, 3]) + #loss = mx.symbol.mean(loss) + return loss + def get_symbol(num_classes): m = config.multiplier - sFilters = max(int(64*m), 32) - mFilters = max(int(128*m), 32) - nFilters = int(256*m) + sFilters = max(int(64 * m), 32) + mFilters = max(int(128 * m), 32) + nFilters = int(256 * m) nModules = 1 nStacks = config.net_stacks @@ -492,80 +888,151 @@ def get_symbol(num_classes): D = input_size // label_size print(input_size, label_size, D) data = mx.sym.Variable(name='data') - data = data-127.5 - data = data*0.0078125 + data = data - 127.5 + data = data * 0.0078125 gt_label = mx.symbol.Variable(name='softmax_label') losses = [] closses = [] ref_label = gt_label - if D==4: - body = Conv(data=data, num_filter=sFilters, kernel=(7, 7), stride=(2,2), pad=(3, 3), - no_bias=True, name="conv0", workspace=workspace) + if D == 4: + body = Conv(data=data, + num_filter=sFilters, + kernel=(7, 7), + stride=(2, 2), + pad=(3, 3), + no_bias=True, + name="conv0", + workspace=workspace) else: - body = Conv(data=data, num_filter=sFilters, kernel=(3, 3), stride=(1,1), pad=(1, 1), - no_bias=True, name="conv0", workspace=workspace) - body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn0') + body = Conv(data=data, + num_filter=sFilters, + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + name="conv0", + workspace=workspace) + body = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name='bn0') body = Act(data=body, act_type='relu', name='relu0') dcn = False - body = conv_block(body, mFilters, (1,1), sFilters==mFilters, 'res0', False, dcn, 1) + body = conv_block(body, mFilters, (1, 1), sFilters == mFilters, 'res0', + False, dcn, 1) - body = mx.sym.Pooling(data=body, kernel=(2, 2), stride=(2,2), pad=(0,0), pool_type='max') + body = mx.sym.Pooling(data=body, + kernel=(2, 2), + stride=(2, 2), + pad=(0, 0), + pool_type='max') - body = conv_block(body, mFilters, (1,1), True, 'res1', False, dcn, 1) #TODO - body = conv_block(body, nFilters, (1,1), mFilters==nFilters, 'res2', binarize, dcn, 1) #binarize=True? + body = conv_block(body, mFilters, (1, 1), True, 'res1', False, dcn, + 1) #TODO + body = conv_block(body, nFilters, (1, 1), mFilters == nFilters, 'res2', + binarize, dcn, 1) #binarize=True? heatmap = None for i in range(nStacks): - shortcut = body - if config.net_sta>0: - sta = STA(body, nFilters, nModules, config.net_n+1, workspace, 'sta%d'%(i)) - body = sta.get() - else: - body = hourglass(body, nFilters, nModules, config.net_n, workspace, 'stack%d_hg'%(i), binarize, dcn) - for j in range(nModules): - body = conv_block(body, nFilters, (1,1), True, 'stack%d_unit%d'%(i,j), binarize, dcn, 1) - _dcn = True if config.net_dcn>=2 else False - ll = ConvFactory(body, nFilters, (1,1), dcn = _dcn, name='stack%d_ll'%(i)) - _name = "heatmap%d"%(i) if i=2 else False - if not _dcn: - out = Conv(data=ll, num_filter=num_classes, kernel=(1, 1), stride=(1,1), pad=(0,0), - name=_name, workspace=workspace) - else: - out_offset = mx.symbol.Convolution(name=_name+'_offset', data = ll, - num_filter=18, pad=(1, 1), kernel=(3, 3), stride=(1, 1)) - out = mx.contrib.symbol.DeformableConvolution(name=_name, data=ll, offset=out_offset, - num_filter=num_classes, pad=(1,1), kernel=(3, 3), num_deformable_group=1, stride=(1, 1), dilate=(1, 1), no_bias=False) - #out = Conv(data=ll, num_filter=num_classes, kernel=(3,3), stride=(1,1), pad=(1,1), - # name=_name, workspace=workspace) - if i==nStacks-1: - heatmap = out - loss = ce_loss(out, ref_label) - #loss = loss/nStacks - #loss = l2_loss(out, ref_label) - losses.append(loss) - if config.net_coherent>0: - ux, dx = coherentor.get(out) - closs = l2_loss(ux, dx) - closs = closs/nStacks - closses.append(closs) + shortcut = body + if config.net_sta > 0: + sta = STA(body, nFilters, nModules, config.net_n + 1, workspace, + 'sta%d' % (i)) + body = sta.get() + else: + body = hourglass(body, nFilters, nModules, config.net_n, workspace, + 'stack%d_hg' % (i), binarize, dcn) + for j in range(nModules): + body = conv_block(body, nFilters, (1, 1), True, + 'stack%d_unit%d' % (i, j), binarize, dcn, 1) + _dcn = True if config.net_dcn >= 2 else False + ll = ConvFactory(body, + nFilters, (1, 1), + dcn=_dcn, + name='stack%d_ll' % (i)) + _name = "heatmap%d" % (i) if i < nStacks - 1 else "heatmap" + _dcn = True if config.net_dcn >= 2 else False + if not _dcn: + out = Conv(data=ll, + num_filter=num_classes, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=_name, + workspace=workspace) + else: + out_offset = mx.symbol.Convolution(name=_name + '_offset', + data=ll, + num_filter=18, + pad=(1, 1), + kernel=(3, 3), + stride=(1, 1)) + out = mx.contrib.symbol.DeformableConvolution( + name=_name, + data=ll, + offset=out_offset, + num_filter=num_classes, + pad=(1, 1), + kernel=(3, 3), + num_deformable_group=1, + stride=(1, 1), + dilate=(1, 1), + no_bias=False) + #out = Conv(data=ll, num_filter=num_classes, kernel=(3,3), stride=(1,1), pad=(1,1), + # name=_name, workspace=workspace) + if i == nStacks - 1: + heatmap = out + loss = ce_loss(out, ref_label) + #loss = loss/nStacks + #loss = l2_loss(out, ref_label) + losses.append(loss) + if config.net_coherent > 0: + ux, dx = coherentor.get(out) + closs = l2_loss(ux, dx) + closs = closs / nStacks + closses.append(closs) - if i0: + loss = mx.symbol.MakeLoss(loss) + syms.append(loss) + if len(closses) > 0: coherent_weight = 0.0001 closs = mx.symbol.add_n(*closses) - closs = mx.symbol.MakeLoss(closs, grad_scale = coherent_weight) + closs = mx.symbol.MakeLoss(closs, grad_scale=coherent_weight) syms.append(closs) syms.append(pred) - sym = mx.symbol.Group( syms ) + sym = mx.symbol.Group(syms) return sym + def init_weights(sym, data_shape_dict): #print('in hg') arg_name = sym.list_arguments() @@ -597,22 +1065,21 @@ def init_weights(sym, data_shape_dict): arg_params = {} aux_params = {} for k in arg_shape_dict: - v = arg_shape_dict[k] - #print(k,v) - if k.endswith('offset_weight') or k.endswith('offset_bias'): - print('initializing',k) - arg_params[k] = mx.nd.zeros(shape = v) - elif k.startswith('fc6_'): - if k.endswith('_weight'): - print('initializing',k) - arg_params[k] = mx.random.normal(0, 0.01, shape=v) - elif k.endswith('_bias'): - print('initializing',k) - arg_params[k] = mx.nd.zeros(shape=v) - elif k.find('upsampling')>=0: - print('initializing upsampling_weight', k) - arg_params[k] = mx.nd.zeros(shape=arg_shape_dict[k]) - init = mx.init.Initializer() - init._init_bilinear(k, arg_params[k]) + v = arg_shape_dict[k] + #print(k,v) + if k.endswith('offset_weight') or k.endswith('offset_bias'): + print('initializing', k) + arg_params[k] = mx.nd.zeros(shape=v) + elif k.startswith('fc6_'): + if k.endswith('_weight'): + print('initializing', k) + arg_params[k] = mx.random.normal(0, 0.01, shape=v) + elif k.endswith('_bias'): + print('initializing', k) + arg_params[k] = mx.nd.zeros(shape=v) + elif k.find('upsampling') >= 0: + print('initializing upsampling_weight', k) + arg_params[k] = mx.nd.zeros(shape=arg_shape_dict[k]) + init = mx.init.Initializer() + init._init_bilinear(k, arg_params[k]) return arg_params, aux_params - diff --git a/alignment/heatmapReg/test.py b/alignment/heatmapReg/test.py index 8c753c3..160e5ac 100644 --- a/alignment/heatmapReg/test.py +++ b/alignment/heatmapReg/test.py @@ -11,55 +11,64 @@ from mtcnn_detector import MtcnnDetector class Handler: - def __init__(self, prefix, epoch, ctx_id=0): - print('loading',prefix, epoch) - if ctx_id>=0: - ctx = mx.gpu(ctx_id) - else: - ctx = mx.cpu() - sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) - all_layers = sym.get_internals() - sym = all_layers['heatmap_output'] - image_size = (128, 128) - self.image_size = image_size - model = mx.mod.Module(symbol=sym, context=ctx, label_names = None) - #model = mx.mod.Module(symbol=sym, context=ctx) - model.bind(for_training=False, data_shapes=[('data', (1, 3, image_size[0], image_size[1]))]) - model.set_params(arg_params, aux_params) - self.model = model - mtcnn_path = os.path.join(os.path.dirname(__file__), '..', 'deploy', 'mtcnn-model') - self.det_threshold = [0.6,0.7,0.8] - self.detector = MtcnnDetector(model_folder=mtcnn_path, ctx=ctx, num_worker=1, accurate_landmark = True, threshold=self.det_threshold) - - def get(self, img): - ret = self.detector.detect_face(img, det_type = 0) - if ret is None: - return None - bbox, points = ret - if bbox.shape[0]==0: - return None - bbox = bbox[0,0:4] - points = points[0,:].reshape((2,5)).T - M = img_helper.estimate_trans_bbox(bbox, self.image_size[0], s = 2.0) - rimg = cv2.warpAffine(img, M, self.image_size, borderValue = 0.0) - img = cv2.cvtColor(rimg, cv2.COLOR_BGR2RGB) - img = np.transpose(img, (2,0,1)) #3*112*112, RGB - input_blob = np.zeros( (1, 3, self.image_size[1], self.image_size[0]),dtype=np.uint8 ) - input_blob[0] = img - ta = datetime.datetime.now() - data = mx.nd.array(input_blob) - db = mx.io.DataBatch(data=(data,)) - self.model.forward(db, is_train=False) - alabel = self.model.get_outputs()[-1].asnumpy()[0] - tb = datetime.datetime.now() - print('module time cost', (tb-ta).total_seconds()) - ret = np.zeros( (alabel.shape[0], 2), dtype=np.float32) - for i in range(alabel.shape[0]): - a = cv2.resize(alabel[i], (self.image_size[1], self.image_size[0])) - ind = np.unravel_index(np.argmax(a, axis=None), a.shape) - #ret[i] = (ind[0], ind[1]) #h, w - ret[i] = (ind[1], ind[0]) #w, h - return ret, M + def __init__(self, prefix, epoch, ctx_id=0): + print('loading', prefix, epoch) + if ctx_id >= 0: + ctx = mx.gpu(ctx_id) + else: + ctx = mx.cpu() + sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) + all_layers = sym.get_internals() + sym = all_layers['heatmap_output'] + image_size = (128, 128) + self.image_size = image_size + model = mx.mod.Module(symbol=sym, context=ctx, label_names=None) + #model = mx.mod.Module(symbol=sym, context=ctx) + model.bind(for_training=False, + data_shapes=[('data', (1, 3, image_size[0], image_size[1])) + ]) + model.set_params(arg_params, aux_params) + self.model = model + mtcnn_path = os.path.join(os.path.dirname(__file__), '..', 'deploy', + 'mtcnn-model') + self.det_threshold = [0.6, 0.7, 0.8] + self.detector = MtcnnDetector(model_folder=mtcnn_path, + ctx=ctx, + num_worker=1, + accurate_landmark=True, + threshold=self.det_threshold) + + def get(self, img): + ret = self.detector.detect_face(img, det_type=0) + if ret is None: + return None + bbox, points = ret + if bbox.shape[0] == 0: + return None + bbox = bbox[0, 0:4] + points = points[0, :].reshape((2, 5)).T + M = img_helper.estimate_trans_bbox(bbox, self.image_size[0], s=2.0) + rimg = cv2.warpAffine(img, M, self.image_size, borderValue=0.0) + img = cv2.cvtColor(rimg, cv2.COLOR_BGR2RGB) + img = np.transpose(img, (2, 0, 1)) #3*112*112, RGB + input_blob = np.zeros((1, 3, self.image_size[1], self.image_size[0]), + dtype=np.uint8) + input_blob[0] = img + ta = datetime.datetime.now() + data = mx.nd.array(input_blob) + db = mx.io.DataBatch(data=(data, )) + self.model.forward(db, is_train=False) + alabel = self.model.get_outputs()[-1].asnumpy()[0] + tb = datetime.datetime.now() + print('module time cost', (tb - ta).total_seconds()) + ret = np.zeros((alabel.shape[0], 2), dtype=np.float32) + for i in range(alabel.shape[0]): + a = cv2.resize(alabel[i], (self.image_size[1], self.image_size[0])) + ind = np.unravel_index(np.argmax(a, axis=None), a.shape) + #ret[i] = (ind[0], ind[1]) #h, w + ret[i] = (ind[1], ind[0]) #w, h + return ret, M + ctx_id = 4 img_path = '../deploy/Tom_Hanks_54745.png' @@ -68,26 +77,24 @@ img = cv2.imread(img_path) handler = Handler('./model/HG', 1, ctx_id) for _ in range(10): - ta = datetime.datetime.now() - landmark, M = handler.get(img) - tb = datetime.datetime.now() - print('get time cost', (tb-ta).total_seconds()) + ta = datetime.datetime.now() + landmark, M = handler.get(img) + tb = datetime.datetime.now() + print('get time cost', (tb - ta).total_seconds()) #visualize landmark IM = cv2.invertAffineTransform(M) for i in range(landmark.shape[0]): - p = landmark[i] - point = np.ones( (3,), dtype=np.float32) - point[0:2] = p - point = np.dot(IM, point) - landmark[i] = point[0:2] + p = landmark[i] + point = np.ones((3, ), dtype=np.float32) + point[0:2] = p + point = np.dot(IM, point) + landmark[i] = point[0:2] for i in range(landmark.shape[0]): - p = landmark[i] - point = (int(p[0]), int(p[1])) - cv2.circle(img, point, 1, (0, 255, 0), 2) + p = landmark[i] + point = (int(p[0]), int(p[1])) + cv2.circle(img, point, 1, (0, 255, 0), 2) filename = './landmark_test.png' print('writing', filename) cv2.imwrite(filename, img) - - diff --git a/alignment/heatmapReg/test_rec_nme.py b/alignment/heatmapReg/test_rec_nme.py index af11971..baad7c7 100644 --- a/alignment/heatmapReg/test_rec_nme.py +++ b/alignment/heatmapReg/test_rec_nme.py @@ -12,7 +12,9 @@ from metric import LossValueMetric, NMEMetric parser = argparse.ArgumentParser(description='test nme on rec data') # general -parser.add_argument('--rec', default='./data_2d/ibug.rec', help='rec data path') +parser.add_argument('--rec', + default='./data_2d/ibug.rec', + help='rec data path') parser.add_argument('--prefix', default='', help='model prefix') parser.add_argument('--epoch', type=int, default=1, help='model epoch') parser.add_argument('--gpu', type=int, default=0, help='') @@ -28,38 +30,42 @@ image_size = (args.image_size, args.image_size) config.landmark_type = args.landmark_type config.input_img_size = image_size[0] -if ctx_id>=0: - ctx = mx.gpu(ctx_id) +if ctx_id >= 0: + ctx = mx.gpu(ctx_id) else: - ctx = mx.cpu() + ctx = mx.cpu() sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) all_layers = sym.get_internals() sym = all_layers['heatmap_output'] #model = mx.mod.Module(symbol=sym, context=ctx, data_names=['data'], label_names=['softmax_label']) -model = mx.mod.Module(symbol=sym, context=ctx, data_names=['data'], label_names=None) +model = mx.mod.Module(symbol=sym, + context=ctx, + data_names=['data'], + label_names=None) #model = mx.mod.Module(symbol=sym, context=ctx) -model.bind(for_training=False, data_shapes=[('data', (1, 3, image_size[0], image_size[1]))]) +model.bind(for_training=False, + data_shapes=[('data', (1, 3, image_size[0], image_size[1]))]) model.set_params(arg_params, aux_params) -val_iter = FaceSegIter(path_imgrec = rec_path, - batch_size = 1, - aug_level = 0, - ) +val_iter = FaceSegIter( + path_imgrec=rec_path, + batch_size=1, + aug_level=0, +) _metric = NMEMetric() #val_metric = mx.metric.create(_metric) #val_metric.reset() #val_iter.reset() nme = [] for i, eval_batch in enumerate(val_iter): - if i%10==0: - print('processing', i) - #print(eval_batch.data[0].shape, eval_batch.label[0].shape) - batch_data = mx.io.DataBatch(eval_batch.data) - model.forward(batch_data, is_train=False) - #model.update_metric(val_metric, eval_batch.label, True) - pred_label = model.get_outputs()[-1].asnumpy() - label = eval_batch.label[0].asnumpy() - _nme = _metric.cal_nme(label, pred_label) - nme.append(_nme) + if i % 10 == 0: + print('processing', i) + #print(eval_batch.data[0].shape, eval_batch.label[0].shape) + batch_data = mx.io.DataBatch(eval_batch.data) + model.forward(batch_data, is_train=False) + #model.update_metric(val_metric, eval_batch.label, True) + pred_label = model.get_outputs()[-1].asnumpy() + label = eval_batch.label[0].asnumpy() + _nme = _metric.cal_nme(label, pred_label) + nme.append(_nme) print(np.mean(nme)) - diff --git a/alignment/heatmapReg/train.py b/alignment/heatmapReg/train.py index c92dcf3..81412e4 100644 --- a/alignment/heatmapReg/train.py +++ b/alignment/heatmapReg/train.py @@ -21,183 +21,216 @@ import sym_heatmap #import sym_fc #from symbol import fc - args = None logger = logging.getLogger() logger.setLevel(logging.INFO) def main(args): - _seed = 727 - random.seed(_seed) - np.random.seed(_seed) - mx.random.seed(_seed) - ctx = [] - cvd = os.environ['CUDA_VISIBLE_DEVICES'].strip() - if len(cvd)>0: - for i in range(len(cvd.split(','))): - ctx.append(mx.gpu(i)) - if len(ctx)==0: - ctx = [mx.cpu()] - print('use cpu') - else: - print('gpu num:', len(ctx)) - #ctx = [mx.gpu(0)] - args.ctx_num = len(ctx) + _seed = 727 + random.seed(_seed) + np.random.seed(_seed) + mx.random.seed(_seed) + ctx = [] + cvd = os.environ['CUDA_VISIBLE_DEVICES'].strip() + if len(cvd) > 0: + for i in range(len(cvd.split(','))): + ctx.append(mx.gpu(i)) + if len(ctx) == 0: + ctx = [mx.cpu()] + print('use cpu') + else: + print('gpu num:', len(ctx)) + #ctx = [mx.gpu(0)] + args.ctx_num = len(ctx) - args.batch_size = args.per_batch_size*args.ctx_num - config.per_batch_size = args.per_batch_size + args.batch_size = args.per_batch_size * args.ctx_num + config.per_batch_size = args.per_batch_size + print('Call with', args, config) + train_iter = FaceSegIter( + path_imgrec=os.path.join(config.dataset_path, 'train.rec'), + batch_size=args.batch_size, + per_batch_size=args.per_batch_size, + aug_level=1, + exf=args.exf, + args=args, + ) + data_shape = train_iter.get_data_shape() + #label_shape = train_iter.get_label_shape() + sym = sym_heatmap.get_symbol(num_classes=config.num_classes) + if len(args.pretrained) == 0: + #data_shape_dict = {'data' : (args.per_batch_size,)+data_shape, 'softmax_label' : (args.per_batch_size,)+label_shape} + data_shape_dict = train_iter.get_shape_dict() + arg_params, aux_params = sym_heatmap.init_weights(sym, data_shape_dict) + else: + vec = args.pretrained.split(',') + print('loading', vec) + _, arg_params, aux_params = mx.model.load_checkpoint( + vec[0], int(vec[1])) + #sym, arg_params, aux_params = get_symbol(args, arg_params, aux_params) - print('Call with', args, config) - train_iter = FaceSegIter(path_imgrec = os.path.join(config.dataset_path, 'train.rec'), - batch_size = args.batch_size, - per_batch_size = args.per_batch_size, - aug_level = 1, - exf = args.exf, - args = args, - ) + model = mx.mod.Module( + context=ctx, + symbol=sym, + label_names=train_iter.get_label_names(), + ) + #lr = 1.0e-3 + #lr = 2.5e-4 + _rescale_grad = 1.0 / args.ctx_num + #_rescale_grad = 1.0/args.batch_size + #lr = args.lr + #opt = optimizer.Nadam(learning_rate=args.lr, wd=args.wd, rescale_grad=_rescale_grad, clip_gradient=5.0) + if args.optimizer == 'onadam': + opt = ONadam(learning_rate=args.lr, + wd=args.wd, + rescale_grad=_rescale_grad, + clip_gradient=5.0) + elif args.optimizer == 'nadam': + opt = optimizer.Nadam(learning_rate=args.lr, + rescale_grad=_rescale_grad) + elif args.optimizer == 'rmsprop': + opt = optimizer.RMSProp(learning_rate=args.lr, + rescale_grad=_rescale_grad) + elif args.optimizer == 'adam': + opt = optimizer.Adam(learning_rate=args.lr, rescale_grad=_rescale_grad) + else: + opt = optimizer.SGD(learning_rate=args.lr, + momentum=0.9, + wd=args.wd, + rescale_grad=_rescale_grad) + initializer = mx.init.Xavier(rnd_type='gaussian', + factor_type="in", + magnitude=2) + _cb = mx.callback.Speedometer(args.batch_size, args.frequent) + _metric = LossValueMetric() + #_metric = NMEMetric() + #_metric2 = AccMetric() + #eval_metrics = [_metric, _metric2] + eval_metrics = [_metric] + lr_steps = [int(x) for x in args.lr_step.split(',')] + print('lr-steps', lr_steps) + global_step = [0] - data_shape = train_iter.get_data_shape() - #label_shape = train_iter.get_label_shape() - sym = sym_heatmap.get_symbol(num_classes=config.num_classes) - if len(args.pretrained)==0: - #data_shape_dict = {'data' : (args.per_batch_size,)+data_shape, 'softmax_label' : (args.per_batch_size,)+label_shape} - data_shape_dict = train_iter.get_shape_dict() - arg_params, aux_params = sym_heatmap.init_weights(sym, data_shape_dict) - else: - vec = args.pretrained.split(',') - print('loading', vec) - _, arg_params, aux_params = mx.model.load_checkpoint(vec[0], int(vec[1])) - #sym, arg_params, aux_params = get_symbol(args, arg_params, aux_params) + def val_test(): + all_layers = sym.get_internals() + vsym = all_layers['heatmap_output'] + vmodel = mx.mod.Module(symbol=vsym, context=ctx, label_names=None) + #model.bind(data_shapes=[('data', (args.batch_size, 3, image_size[0], image_size[1]))], label_shapes=[('softmax_label', (args.batch_size,))]) + vmodel.bind(data_shapes=[('data', (args.batch_size, ) + data_shape)]) + arg_params, aux_params = model.get_params() + vmodel.set_params(arg_params, aux_params) + for target in config.val_targets: + _file = os.path.join(config.dataset_path, '%s.rec' % target) + if not os.path.exists(_file): + continue + val_iter = FaceSegIter( + path_imgrec=_file, + batch_size=args.batch_size, + #batch_size = 4, + aug_level=0, + args=args, + ) + _metric = NMEMetric() + val_metric = mx.metric.create(_metric) + val_metric.reset() + val_iter.reset() + for i, eval_batch in enumerate(val_iter): + #print(eval_batch.data[0].shape, eval_batch.label[0].shape) + batch_data = mx.io.DataBatch(eval_batch.data) + model.forward(batch_data, is_train=False) + model.update_metric(val_metric, eval_batch.label) + nme_value = val_metric.get_name_value()[0][1] + print('[%d][%s]NME: %f' % (global_step[0], target, nme_value)) - model = mx.mod.Module( - context = ctx, - symbol = sym, - label_names = train_iter.get_label_names(), - ) - #lr = 1.0e-3 - #lr = 2.5e-4 - _rescale_grad = 1.0/args.ctx_num - #_rescale_grad = 1.0/args.batch_size - #lr = args.lr - #opt = optimizer.Nadam(learning_rate=args.lr, wd=args.wd, rescale_grad=_rescale_grad, clip_gradient=5.0) - if args.optimizer=='onadam': - opt = ONadam(learning_rate=args.lr, wd=args.wd, rescale_grad=_rescale_grad, clip_gradient=5.0) - elif args.optimizer=='nadam': - opt = optimizer.Nadam(learning_rate=args.lr, rescale_grad=_rescale_grad) - elif args.optimizer=='rmsprop': - opt = optimizer.RMSProp(learning_rate=args.lr, rescale_grad=_rescale_grad) - elif args.optimizer=='adam': - opt = optimizer.Adam(learning_rate=args.lr, rescale_grad=_rescale_grad) - else: - opt = optimizer.SGD(learning_rate=args.lr, momentum=0.9, wd=args.wd, rescale_grad=_rescale_grad) - initializer = mx.init.Xavier(rnd_type='gaussian', factor_type="in", magnitude=2) - _cb = mx.callback.Speedometer(args.batch_size, args.frequent) - _metric = LossValueMetric() - #_metric = NMEMetric() - #_metric2 = AccMetric() - #eval_metrics = [_metric, _metric2] - eval_metrics = [_metric] - lr_steps = [int(x) for x in args.lr_step.split(',')] - print('lr-steps', lr_steps) - global_step = [0] + def _batch_callback(param): + _cb(param) + global_step[0] += 1 + mbatch = global_step[0] + for _lr in lr_steps: + if mbatch == _lr: + opt.lr *= 0.2 + print('lr change to', opt.lr) + break + if mbatch % 1000 == 0: + print('lr-batch-epoch:', opt.lr, param.nbatch, param.epoch) + if mbatch > 0 and mbatch % args.verbose == 0: + val_test() + if args.ckpt == 1: + msave = mbatch // args.verbose + print('saving', msave) + arg, aux = model.get_params() + mx.model.save_checkpoint(args.prefix, msave, model.symbol, arg, + aux) + if mbatch == lr_steps[-1]: + if args.ckpt == 2: + #msave = mbatch//args.verbose + msave = 1 + print('saving', msave) + arg, aux = model.get_params() + mx.model.save_checkpoint(args.prefix, msave, model.symbol, arg, + aux) + sys.exit(0) - def val_test(): - all_layers = sym.get_internals() - vsym = all_layers['heatmap_output'] - vmodel = mx.mod.Module(symbol=vsym, context=ctx, label_names = None) - #model.bind(data_shapes=[('data', (args.batch_size, 3, image_size[0], image_size[1]))], label_shapes=[('softmax_label', (args.batch_size,))]) - vmodel.bind(data_shapes=[('data', (args.batch_size,)+data_shape)]) - arg_params, aux_params = model.get_params() - vmodel.set_params(arg_params, aux_params) - for target in config.val_targets: - _file = os.path.join(config.dataset_path, '%s.rec'%target) - if not os.path.exists(_file): - continue - val_iter = FaceSegIter(path_imgrec = _file, - batch_size = args.batch_size, - #batch_size = 4, - aug_level = 0, - args = args, - ) - _metric = NMEMetric() - val_metric = mx.metric.create(_metric) - val_metric.reset() - val_iter.reset() - for i, eval_batch in enumerate(val_iter): - #print(eval_batch.data[0].shape, eval_batch.label[0].shape) - batch_data = mx.io.DataBatch(eval_batch.data) - model.forward(batch_data, is_train=False) - model.update_metric(val_metric, eval_batch.label) - nme_value = val_metric.get_name_value()[0][1] - print('[%d][%s]NME: %f'%(global_step[0], target, nme_value)) - - def _batch_callback(param): - _cb(param) - global_step[0]+=1 - mbatch = global_step[0] - for _lr in lr_steps: - if mbatch==_lr: - opt.lr *= 0.2 - print('lr change to', opt.lr) - break - if mbatch%1000==0: - print('lr-batch-epoch:',opt.lr,param.nbatch,param.epoch) - if mbatch>0 and mbatch%args.verbose==0: - val_test() - if args.ckpt==1: - msave = mbatch//args.verbose - print('saving', msave) - arg, aux = model.get_params() - mx.model.save_checkpoint(args.prefix, msave, model.symbol, arg, aux) - if mbatch==lr_steps[-1]: - if args.ckpt==2: - #msave = mbatch//args.verbose - msave = 1 - print('saving', msave) - arg, aux = model.get_params() - mx.model.save_checkpoint(args.prefix, msave, model.symbol, arg, aux) - sys.exit(0) + train_iter = mx.io.PrefetchingIter(train_iter) - train_iter = mx.io.PrefetchingIter(train_iter) + model.fit( + train_iter, + begin_epoch=0, + num_epoch=9999, + #eval_data = val_iter, + eval_data=None, + eval_metric=eval_metrics, + kvstore='device', + optimizer=opt, + initializer=initializer, + arg_params=arg_params, + aux_params=aux_params, + allow_missing=True, + batch_end_callback=_batch_callback, + epoch_end_callback=None, + ) - model.fit(train_iter, - begin_epoch = 0, - num_epoch = 9999, - #eval_data = val_iter, - eval_data = None, - eval_metric = eval_metrics, - kvstore = 'device', - optimizer = opt, - initializer = initializer, - arg_params = arg_params, - aux_params = aux_params, - allow_missing = True, - batch_end_callback = _batch_callback, - epoch_end_callback = None, - ) if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Train face alignment') - # general - parser.add_argument('--network', help='network name', default=default.network, type=str) - parser.add_argument('--dataset', help='dataset name', default=default.dataset, type=str) - args, rest = parser.parse_known_args() - generate_config(args.network, args.dataset) - parser.add_argument('--prefix', default=default.prefix, help='directory to save model.') - parser.add_argument('--pretrained', default=default.pretrained, help='') - parser.add_argument('--optimizer', default='nadam', help='') - parser.add_argument('--lr', type=float, default=default.lr, help='') - parser.add_argument('--wd', type=float, default=default.wd, help='') - parser.add_argument('--per-batch-size', type=int, default=default.per_batch_size, help='') - parser.add_argument('--lr-step', help='learning rate steps (in epoch)', default=default.lr_step, type=str) - parser.add_argument('--ckpt', type=int, default=1, help='') - parser.add_argument('--norm', type=int, default=0, help='') - parser.add_argument('--exf', type=int, default=1, help='') - parser.add_argument('--frequent', type=int, default=default.frequent, help='') - parser.add_argument('--verbose', type=int, default=default.verbose, help='') - args = parser.parse_args() - main(args) - + parser = argparse.ArgumentParser(description='Train face alignment') + # general + parser.add_argument('--network', + help='network name', + default=default.network, + type=str) + parser.add_argument('--dataset', + help='dataset name', + default=default.dataset, + type=str) + args, rest = parser.parse_known_args() + generate_config(args.network, args.dataset) + parser.add_argument('--prefix', + default=default.prefix, + help='directory to save model.') + parser.add_argument('--pretrained', default=default.pretrained, help='') + parser.add_argument('--optimizer', default='nadam', help='') + parser.add_argument('--lr', type=float, default=default.lr, help='') + parser.add_argument('--wd', type=float, default=default.wd, help='') + parser.add_argument('--per-batch-size', + type=int, + default=default.per_batch_size, + help='') + parser.add_argument('--lr-step', + help='learning rate steps (in epoch)', + default=default.lr_step, + type=str) + parser.add_argument('--ckpt', type=int, default=1, help='') + parser.add_argument('--norm', type=int, default=0, help='') + parser.add_argument('--exf', type=int, default=1, help='') + parser.add_argument('--frequent', + type=int, + default=default.frequent, + help='') + parser.add_argument('--verbose', + type=int, + default=default.verbose, + help='') + args = parser.parse_args() + main(args) diff --git a/IFRT/README.md b/challenges/IFRT/README.md similarity index 100% rename from IFRT/README.md rename to challenges/IFRT/README.md diff --git a/iccv19-challenge/README.md b/challenges/iccv19-lfr/README.md similarity index 100% rename from iccv19-challenge/README.md rename to challenges/iccv19-lfr/README.md diff --git a/challenges/iccv19-lfr/gen_image_feature.py b/challenges/iccv19-lfr/gen_image_feature.py new file mode 100644 index 0000000..c555227 --- /dev/null +++ b/challenges/iccv19-lfr/gen_image_feature.py @@ -0,0 +1,157 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os + +from datetime import datetime +import os.path +from easydict import EasyDict as edict +import time +import json +import sys +import numpy as np +import importlib +import itertools +import argparse +import struct +import cv2 +import sklearn +from sklearn.preprocessing import normalize +import mxnet as mx +from mxnet import ndarray as nd + +image_shape = None +net = None +data_size = 1862120 +emb_size = 0 +use_flip = True + + +def do_flip(data): + for idx in range(data.shape[0]): + data[idx, :, :] = np.fliplr(data[idx, :, :]) + + +def get_feature(buffer): + global emb_size + if use_flip: + input_blob = np.zeros( + (len(buffer) * 2, 3, image_shape[1], image_shape[2])) + else: + input_blob = np.zeros((len(buffer), 3, image_shape[1], image_shape[2])) + idx = 0 + for item in buffer: + img = cv2.imread(item)[:, :, ::-1] #to rgb + img = np.transpose(img, (2, 0, 1)) + attempts = [0, 1] if use_flip else [0] + for flipid in attempts: + _img = np.copy(img) + if flipid == 1: + do_flip(_img) + input_blob[idx] = _img + idx += 1 + data = mx.nd.array(input_blob) + db = mx.io.DataBatch(data=(data, )) + net.model.forward(db, is_train=False) + _embedding = net.model.get_outputs()[0].asnumpy() + if emb_size == 0: + emb_size = _embedding.shape[1] + print('set emb_size to ', emb_size) + embedding = np.zeros((len(buffer), emb_size), dtype=np.float32) + if use_flip: + embedding1 = _embedding[0::2] + embedding2 = _embedding[1::2] + embedding = embedding1 + embedding2 + else: + embedding = _embedding + embedding = sklearn.preprocessing.normalize(embedding) + return embedding + + +def write_bin(path, m): + rows, cols = m.shape + with open(path, 'wb') as f: + f.write(struct.pack('4i', rows, cols, cols * 4, 5)) + f.write(m.data) + + +def main(args): + global image_shape + global net + + print(args) + ctx = [] + cvd = os.environ['CUDA_VISIBLE_DEVICES'].strip() + if len(cvd) > 0: + for i in range(len(cvd.split(','))): + ctx.append(mx.gpu(i)) + if len(ctx) == 0: + ctx = [mx.cpu()] + print('use cpu') + else: + print('gpu num:', len(ctx)) + image_shape = [int(x) for x in args.image_size.split(',')] + vec = args.model.split(',') + assert len(vec) > 1 + prefix = vec[0] + epoch = int(vec[1]) + print('loading', prefix, epoch) + net = edict() + net.ctx = ctx + net.sym, net.arg_params, net.aux_params = mx.model.load_checkpoint( + prefix, epoch) + #net.arg_params, net.aux_params = ch_dev(net.arg_params, net.aux_params, net.ctx) + all_layers = net.sym.get_internals() + net.sym = all_layers['fc1_output'] + net.model = mx.mod.Module(symbol=net.sym, + context=net.ctx, + label_names=None) + net.model.bind(data_shapes=[('data', (args.batch_size, 3, image_shape[1], + image_shape[2]))]) + net.model.set_params(net.arg_params, net.aux_params) + + features_all = None + + i = 0 + fstart = 0 + buffer = [] + for line in open(os.path.join(args.input, 'filelist.txt'), 'r'): + if i % 1000 == 0: + print("processing ", i) + i += 1 + line = line.strip() + image_path = os.path.join(args.input, line) + buffer.append(image_path) + if len(buffer) == args.batch_size: + embedding = get_feature(buffer) + buffer = [] + fend = fstart + embedding.shape[0] + if features_all is None: + features_all = np.zeros((data_size, emb_size), + dtype=np.float32) + #print('writing', fstart, fend) + features_all[fstart:fend, :] = embedding + fstart = fend + if len(buffer) > 0: + embedding = get_feature(buffer) + fend = fstart + embedding.shape[0] + print('writing', fstart, fend) + features_all[fstart:fend, :] = embedding + write_bin(args.output, features_all) + #os.system("bypy upload %s"%args.output) + + +def parse_arguments(argv): + parser = argparse.ArgumentParser() + + parser.add_argument('--batch_size', type=int, help='', default=32) + parser.add_argument('--image_size', type=str, help='', default='3,112,112') + parser.add_argument('--input', type=str, help='', default='') + parser.add_argument('--output', type=str, help='', default='') + parser.add_argument('--model', type=str, help='', default='') + return parser.parse_args(argv) + + +if __name__ == '__main__': + main(parse_arguments(sys.argv[1:])) diff --git a/challenges/iccv19-lfr/gen_video_feature.py b/challenges/iccv19-lfr/gen_video_feature.py new file mode 100644 index 0000000..8b3d7fc --- /dev/null +++ b/challenges/iccv19-lfr/gen_video_feature.py @@ -0,0 +1,215 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os + +from datetime import datetime +import os.path +from easydict import EasyDict as edict +import time +import json +import glob +import sys +import numpy as np +import importlib +import itertools +import argparse +import struct +import cv2 +import sklearn +from sklearn.preprocessing import normalize +import mxnet as mx +from mxnet import ndarray as nd + +image_shape = None +net = None +data_size = 203848 +emb_size = 0 +use_flip = False +ctx_num = 0 + + +def do_flip(data): + for idx in range(data.shape[0]): + data[idx, :, :] = np.fliplr(data[idx, :, :]) + + +def get_feature(buffer): + global emb_size + input_count = len(buffer) + if use_flip: + input_count *= 2 + network_count = input_count + if input_count % ctx_num != 0: + network_count = (input_count // ctx_num + 1) * ctx_num + + input_blob = np.zeros((network_count, 3, image_shape[1], image_shape[2]), + dtype=np.float32) + idx = 0 + for item in buffer: + img = cv2.imread(item)[:, :, ::-1] #to rgb + img = np.transpose(img, (2, 0, 1)) + attempts = [0, 1] if use_flip else [0] + for flipid in attempts: + _img = np.copy(img) + if flipid == 1: + do_flip(_img) + input_blob[idx] = _img + idx += 1 + data = mx.nd.array(input_blob) + db = mx.io.DataBatch(data=(data, )) + net.model.forward(db, is_train=False) + _embedding = net.model.get_outputs()[0].asnumpy() + _embedding = _embedding[0:input_count] + if emb_size == 0: + emb_size = _embedding.shape[1] + print('set emb_size to ', emb_size) + embedding = np.zeros((len(buffer), emb_size), dtype=np.float32) + if use_flip: + embedding1 = _embedding[0::2] + embedding2 = _embedding[1::2] + embedding = embedding1 + embedding2 + else: + embedding = _embedding + embedding = sklearn.preprocessing.normalize(embedding) + return embedding + + +def write_bin(path, m): + rows, cols = m.shape + with open(path, 'wb') as f: + f.write(struct.pack('4i', rows, cols, cols * 4, 5)) + f.write(m.data) + + +def main(args): + global image_shape + global net + global ctx_num + + print(args) + ctx = [] + cvd = os.environ['CUDA_VISIBLE_DEVICES'].strip() + if len(cvd) > 0: + for i in range(len(cvd.split(','))): + ctx.append(mx.gpu(i)) + if len(ctx) == 0: + ctx = [mx.cpu()] + print('use cpu') + else: + print('gpu num:', len(ctx)) + ctx_num = len(ctx) + image_shape = [int(x) for x in args.image_size.split(',')] + vec = args.model.split(',') + assert len(vec) > 1 + prefix = vec[0] + epoch = int(vec[1]) + print('loading', prefix, epoch) + net = edict() + net.ctx = ctx + net.sym, net.arg_params, net.aux_params = mx.model.load_checkpoint( + prefix, epoch) + #net.arg_params, net.aux_params = ch_dev(net.arg_params, net.aux_params, net.ctx) + all_layers = net.sym.get_internals() + net.sym = all_layers['fc1_output'] + net.model = mx.mod.Module(symbol=net.sym, + context=net.ctx, + label_names=None) + net.model.bind(data_shapes=[('data', (args.batch_size, 3, image_shape[1], + image_shape[2]))]) + net.model.set_params(net.arg_params, net.aux_params) + + features_all = None + + i = 0 + filelist = os.path.join(args.input, 'filelist.txt') + #print(filelist) + buffer_images = [] + buffer_embedding = np.zeros((0, 0), dtype=np.float32) + aggr_nums = [] + row_idx = 0 + for line in open(filelist, 'r'): + if i % 1000 == 0: + print("processing ", i) + i += 1 + #print('stat', i, len(buffer_images), buffer_embedding.shape, aggr_nums, row_idx) + videoname = line.strip().split()[0] + images = glob.glob("%s/%s/*.jpg" % (args.input, videoname)) + assert len(images) > 0 + image_features = [] + for image_path in images: + buffer_images.append(image_path) + aggr_nums.append(len(images)) + while len(buffer_images) >= args.batch_size: + embedding = get_feature(buffer_images[0:args.batch_size]) + buffer_images = buffer_images[args.batch_size:] + if buffer_embedding.shape[1] == 0: + buffer_embedding = embedding.copy() + else: + buffer_embedding = np.concatenate( + (buffer_embedding, embedding), axis=0) + buffer_idx = 0 + acount = 0 + for anum in aggr_nums: + if buffer_embedding.shape[0] >= anum + buffer_idx: + image_features = buffer_embedding[buffer_idx:buffer_idx + anum] + video_feature = np.sum(image_features, axis=0, keepdims=True) + video_feature = sklearn.preprocessing.normalize(video_feature) + if features_all is None: + features_all = np.zeros( + (data_size, video_feature.shape[1]), dtype=np.float32) + #print('write to', row_idx, anum, buffer_embedding.shape) + features_all[row_idx] = video_feature.flatten() + row_idx += 1 + buffer_idx += anum + acount += 1 + else: + break + aggr_nums = aggr_nums[acount:] + buffer_embedding = buffer_embedding[buffer_idx:] + + if len(buffer_images) > 0: + embedding = get_feature(buffer_images) + buffer_images = buffer_images[args.batch_size:] + buffer_embedding = np.concatenate((buffer_embedding, embedding), + axis=0) + buffer_idx = 0 + acount = 0 + for anum in aggr_nums: + assert buffer_embedding.shape[0] >= anum + buffer_idx + image_features = buffer_embedding[buffer_idx:buffer_idx + anum] + video_feature = np.sum(image_features, axis=0, keepdims=True) + video_feature = sklearn.preprocessing.normalize(video_feature) + #print('last write to', row_idx, anum, buffer_embedding.shape) + features_all[row_idx] = video_feature.flatten() + row_idx += 1 + buffer_idx += anum + acount += 1 + + aggr_nums = aggr_nums[acount:] + buffer_embedding = buffer_embedding[buffer_idx:] + assert len(aggr_nums) == 0 + assert buffer_embedding.shape[0] == 0 + + write_bin(args.output, features_all) + print(row_idx, features_all.shape) + #os.system("bypy upload %s"%args.output) + + +def parse_arguments(argv): + parser = argparse.ArgumentParser() + + parser.add_argument('--batch_size', type=int, help='', default=32) + parser.add_argument('--image_size', type=str, help='', default='3,112,112') + parser.add_argument('--input', + type=str, + help='', + default='./testdata-video') + parser.add_argument('--output', type=str, help='', default='') + parser.add_argument('--model', type=str, help='', default='') + return parser.parse_args(argv) + + +if __name__ == '__main__': + main(parse_arguments(sys.argv[1:])) diff --git a/common/face_align.py b/common/face_align.py deleted file mode 100644 index 6cf5a2e..0000000 --- a/common/face_align.py +++ /dev/null @@ -1,88 +0,0 @@ - -import cv2 -import numpy as np -from skimage import transform as trans - -src1 = np.array([ - [51.642,50.115], - [57.617,49.990], - [35.740,69.007], - [51.157,89.050], - [57.025,89.702]], dtype=np.float32) -#<--left -src2 = np.array([ - [45.031,50.118], - [65.568,50.872], - [39.677,68.111], - [45.177,86.190], - [64.246,86.758]], dtype=np.float32) - -#---frontal -src3 = np.array([ - [39.730,51.138], - [72.270,51.138], - [56.000,68.493], - [42.463,87.010], - [69.537,87.010]], dtype=np.float32) - -#-->right -src4 = np.array([ - [46.845,50.872], - [67.382,50.118], - [72.737,68.111], - [48.167,86.758], - [67.236,86.190]], dtype=np.float32) - -#-->right profile -src5 = np.array([ - [54.796,49.990], - [60.771,50.115], - [76.673,69.007], - [55.388,89.702], - [61.257,89.050]], dtype=np.float32) - -src = np.array([src1,src2,src3,src4,src5]) -src_map = {112 : src, 224 : src*2} - -arcface_src = np.array([ - [38.2946, 51.6963], - [73.5318, 51.5014], - [56.0252, 71.7366], - [41.5493, 92.3655], - [70.7299, 92.2041] ], dtype=np.float32 ) - -arcface_src = np.expand_dims(arcface_src, axis=0) - -# In[66]: - -# lmk is prediction; src is template -def estimate_norm(lmk, image_size = 112, mode='arcface'): - assert lmk.shape==(5,2) - tform = trans.SimilarityTransform() - lmk_tran = np.insert(lmk, 2, values=np.ones(5), axis=1) - min_M = [] - min_index = [] - min_error = float('inf') - if mode=='arcface': - assert image_size==112 - src = arcface_src - else: - src = src_map[image_size] - for i in np.arange(src.shape[0]): - tform.estimate(lmk, src[i]) - M = tform.params[0:2,:] - results = np.dot(M, lmk_tran.T) - results = results.T - error = np.sum(np.sqrt(np.sum((results - src[i]) ** 2,axis=1))) -# print(error) - if error< min_error: - min_error = error - min_M = M - min_index = i - return min_M, min_index - -def norm_crop(img, landmark, image_size=112, mode='arcface'): - M, pose_index = estimate_norm(landmark, image_size, mode) - warped = cv2.warpAffine(img,M, (image_size, image_size), borderValue = 0.0) - return warped - diff --git a/common/flops_counter.py b/common/flops_counter.py deleted file mode 100644 index 64469ec..0000000 --- a/common/flops_counter.py +++ /dev/null @@ -1,114 +0,0 @@ - -''' -@author: insightface -''' - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import sys -import os -import json -import argparse -import numpy as np -import mxnet as mx - - -def is_no_bias(attr): - ret = False - if 'no_bias' in attr and (attr['no_bias']==True or attr['no_bias']=='True'): - ret = True - return ret - -def count_fc_flops(input_filter, output_filter, attr): - #print(input_filter, output_filter ,attr) - ret = 2*input_filter*output_filter - if is_no_bias(attr): - ret -= output_filter - return int(ret) - - -def count_conv_flops(input_shape, output_shape, attr): - kernel = attr['kernel'][1:-1].split(',') - kernel = [int(x) for x in kernel] - - #print('kernel', kernel) - if is_no_bias(attr): - ret = (2*input_shape[1]*kernel[0]*kernel[1]-1)*output_shape[2]*output_shape[3]*output_shape[1] - else: - ret = 2*input_shape[1]*kernel[0]*kernel[1]*output_shape[2]*output_shape[3]*output_shape[1] - num_group = 1 - if 'num_group' in attr: - num_group = int(attr['num_group']) - ret /= num_group - return int(ret) - - -def count_flops(sym, **data_shapes): - all_layers = sym.get_internals() - #print(all_layers) - arg_shapes, out_shapes, aux_shapes = all_layers.infer_shape(**data_shapes) - out_shape_dict = dict(zip(all_layers.list_outputs(), out_shapes)) - - nodes = json.loads(sym.tojson())['nodes'] - nodeid_shape = {} - for nodeid, node in enumerate(nodes): - name = node['name'] - layer_name = name+"_output" - if layer_name in out_shape_dict: - nodeid_shape[nodeid] = out_shape_dict[layer_name] - #print(nodeid_shape) - FLOPs = 0 - for nodeid, node in enumerate(nodes): - flops = 0 - if node['op']=='Convolution': - output_shape = nodeid_shape[nodeid] - name = node['name'] - attr = node['attrs'] - input_nodeid = node['inputs'][0][0] - input_shape = nodeid_shape[input_nodeid] - flops = count_conv_flops(input_shape, output_shape, attr) - elif node['op']=='FullyConnected': - attr = node['attrs'] - output_shape = nodeid_shape[nodeid] - input_nodeid = node['inputs'][0][0] - input_shape = nodeid_shape[input_nodeid] - output_filter = output_shape[1] - input_filter = input_shape[1]*input_shape[2]*input_shape[3] - #assert len(input_shape)==4 and input_shape[2]==1 and input_shape[3]==1 - flops = count_fc_flops(input_filter, output_filter, attr) - #print(node, flops) - FLOPs += flops - - return FLOPs - -def flops_str(FLOPs): - preset = [ (1e12, 'T'), (1e9, 'G'), (1e6, 'M'), (1e3, 'K') ] - - for p in preset: - if FLOPs//p[0]>0: - N = FLOPs/p[0] - ret = "%.1f%s"%(N, p[1]) - return ret - ret = "%.1f"%(FLOPs) - return ret - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='flops counter') - # general - #parser.add_argument('--model', default='../models2/y2-arcface-retinat1/model,1', help='path to load model.') - #parser.add_argument('--model', default='../models2/r100fc-arcface-retinaa/model,1', help='path to load model.') - parser.add_argument('--model', default='../models2/r50fc-arcface-emore/model,1', help='path to load model.') - args = parser.parse_args() - _vec = args.model.split(',') - assert len(_vec)==2 - prefix = _vec[0] - epoch = int(_vec[1]) - print('loading',prefix, epoch) - sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) - all_layers = sym.get_internals() - sym = all_layers['fc1_output'] - FLOPs = count_flops(sym, data=(1,3,112,112)) - print('FLOPs:', FLOPs) - diff --git a/datasets/README.md b/datasets/README.md deleted file mode 100644 index 828aedd..0000000 --- a/datasets/README.md +++ /dev/null @@ -1 +0,0 @@ -Put datasets here. diff --git a/deploy/benchmark.py b/deploy/benchmark.py index 42773b0..2f2e8bc 100644 --- a/deploy/benchmark.py +++ b/deploy/benchmark.py @@ -7,20 +7,33 @@ import datetime parser = argparse.ArgumentParser(description='face model test') # general parser.add_argument('--image-size', default='112,112', help='') -parser.add_argument('--model', default='../models/model-r34-amf/model,0', help='path to load model.') +parser.add_argument('--model', + default='../models/model-r34-amf/model,0', + help='path to load model.') parser.add_argument('--gpu', default=0, type=int, help='gpu id') -parser.add_argument('--det', default=2, type=int, help='mtcnn option, 2 means using R+O, else using O') -parser.add_argument('--flip', default=0, type=int, help='whether do lr flip aug') -parser.add_argument('--threshold', default=1.24, type=float, help='ver dist threshold') +parser.add_argument('--det', + default=2, + type=int, + help='mtcnn option, 2 means using R+O, else using O') +parser.add_argument('--flip', + default=0, + type=int, + help='whether do lr flip aug') +parser.add_argument('--threshold', + default=1.24, + type=float, + help='ver dist threshold') args = parser.parse_args() model = face_embedding.FaceModel(args) #img = cv2.imread('/raid5data/dplearn/lfw/Jude_Law/Jude_Law_0001.jpg') -img = cv2.imread('/raid5data/dplearn/megaface/facescrubr/112x112/Tom_Hanks/Tom_Hanks_54745.png') +img = cv2.imread( + '/raid5data/dplearn/megaface/facescrubr/112x112/Tom_Hanks/Tom_Hanks_54745.png' +) time_now = datetime.datetime.now() for i in range(3000): - f1 = model.get_feature(img) + f1 = model.get_feature(img) time_now2 = datetime.datetime.now() diff = time_now2 - time_now -print(diff.total_seconds()/3000) +print(diff.total_seconds() / 3000) diff --git a/deploy/convert_onnx.py b/deploy/convert_onnx.py index 75d4f67..3ed583d 100644 --- a/deploy/convert_onnx.py +++ b/deploy/convert_onnx.py @@ -9,23 +9,32 @@ print('onnx version:', onnx.__version__) #make sure to install onnx-1.2.1 #pip uninstall onnx #pip install onnx==1.2.1 -assert onnx.__version__=='1.2.1' +assert onnx.__version__ == '1.2.1' import numpy as np from mxnet.contrib import onnx as onnx_mxnet -parser = argparse.ArgumentParser(description='convert insightface models to onnx') +parser = argparse.ArgumentParser( + description='convert insightface models to onnx') # general -parser.add_argument('--prefix', default='./r100-arcface/model', help='prefix to load model.') -parser.add_argument('--epoch', default=0, type=int, help='epoch number to load model.') +parser.add_argument('--prefix', + default='./r100-arcface/model', + help='prefix to load model.') +parser.add_argument('--epoch', + default=0, + type=int, + help='epoch number to load model.') parser.add_argument('--input-shape', default='3,112,112', help='input shape.') -parser.add_argument('--output-onnx', default='./r100.onnx', help='path to write onnx model.') +parser.add_argument('--output-onnx', + default='./r100.onnx', + help='path to write onnx model.') args = parser.parse_args() -input_shape = (1,) + tuple( [int(x) for x in args.input_shape.split(',')] ) +input_shape = (1, ) + tuple([int(x) for x in args.input_shape.split(',')]) print('input-shape:', input_shape) -sym_file = "%s-symbol.json"%args.prefix -params_file = "%s-%04d.params"%(args.prefix, args.epoch) +sym_file = "%s-symbol.json" % args.prefix +params_file = "%s-%04d.params" % (args.prefix, args.epoch) assert os.path.exists(sym_file) assert os.path.exists(params_file) -converted_model_path = onnx_mxnet.export_model(sym_file, params_file, [input_shape], np.float32, args.output_onnx) - +converted_model_path = onnx_mxnet.export_model(sym_file, params_file, + [input_shape], np.float32, + args.output_onnx) diff --git a/deploy/face_embedding.py b/deploy/face_embedding.py index ae2dd1c..9b81632 100644 --- a/deploy/face_embedding.py +++ b/deploy/face_embedding.py @@ -22,73 +22,81 @@ import face_preprocess def do_flip(data): - for idx in range(data.shape[0]): - data[idx,:,:] = np.fliplr(data[idx,:,:]) + for idx in range(data.shape[0]): + data[idx, :, :] = np.fliplr(data[idx, :, :]) + class FaceModel: - def __init__(self, args): - self.args = args - model = edict() + def __init__(self, args): + self.args = args + model = edict() - self.threshold = args.threshold - self.det_minsize = 50 - self.det_threshold = [0.4,0.6,0.6] - self.det_factor = 0.9 - _vec = args.image_size.split(',') - assert len(_vec)==2 - image_size = (int(_vec[0]), int(_vec[1])) - self.image_size = image_size - _vec = args.model.split(',') - assert len(_vec)==2 - prefix = _vec[0] - epoch = int(_vec[1]) - print('loading',prefix, epoch) - ctx = mx.gpu(args.gpu) - sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) - all_layers = sym.get_internals() - sym = all_layers['fc1_output'] - model = mx.mod.Module(symbol=sym, context=ctx, label_names = None) - #model.bind(data_shapes=[('data', (args.batch_size, 3, image_size[0], image_size[1]))], label_shapes=[('softmax_label', (args.batch_size,))]) - model.bind(data_shapes=[('data', (1, 3, image_size[0], image_size[1]))]) - model.set_params(arg_params, aux_params) - self.model = model - mtcnn_path = os.path.join(os.path.dirname(__file__), 'mtcnn-model') - detector = MtcnnDetector(model_folder=mtcnn_path, ctx=ctx, num_worker=1, accurate_landmark = True, threshold=[0.0,0.0,0.2]) - self.detector = detector - - - def get_feature(self, face_img): - #face_img is bgr image - ret = self.detector.detect_face_limited(face_img, det_type = self.args.det) - if ret is None: - return None - bbox, points = ret - if bbox.shape[0]==0: - return None - bbox = bbox[0,0:4] - points = points[0,:].reshape((2,5)).T - #print(bbox) - #print(points) - nimg = face_preprocess.preprocess(face_img, bbox, points, image_size='112,112') - nimg = cv2.cvtColor(nimg, cv2.COLOR_BGR2RGB) - aligned = np.transpose(nimg, (2,0,1)) - #print(nimg.shape) - embedding = None - for flipid in [0,1]: - if flipid==1: - if self.args.flip==0: - break - do_flip(aligned) - input_blob = np.expand_dims(aligned, axis=0) - data = mx.nd.array(input_blob) - db = mx.io.DataBatch(data=(data,)) - self.model.forward(db, is_train=False) - _embedding = self.model.get_outputs()[0].asnumpy() - #print(_embedding.shape) - if embedding is None: - embedding = _embedding - else: - embedding += _embedding - embedding = sklearn.preprocessing.normalize(embedding).flatten() - return embedding + self.threshold = args.threshold + self.det_minsize = 50 + self.det_threshold = [0.4, 0.6, 0.6] + self.det_factor = 0.9 + _vec = args.image_size.split(',') + assert len(_vec) == 2 + image_size = (int(_vec[0]), int(_vec[1])) + self.image_size = image_size + _vec = args.model.split(',') + assert len(_vec) == 2 + prefix = _vec[0] + epoch = int(_vec[1]) + print('loading', prefix, epoch) + ctx = mx.gpu(args.gpu) + sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) + all_layers = sym.get_internals() + sym = all_layers['fc1_output'] + model = mx.mod.Module(symbol=sym, context=ctx, label_names=None) + #model.bind(data_shapes=[('data', (args.batch_size, 3, image_size[0], image_size[1]))], label_shapes=[('softmax_label', (args.batch_size,))]) + model.bind(data_shapes=[('data', (1, 3, image_size[0], + image_size[1]))]) + model.set_params(arg_params, aux_params) + self.model = model + mtcnn_path = os.path.join(os.path.dirname(__file__), 'mtcnn-model') + detector = MtcnnDetector(model_folder=mtcnn_path, + ctx=ctx, + num_worker=1, + accurate_landmark=True, + threshold=[0.0, 0.0, 0.2]) + self.detector = detector + def get_feature(self, face_img): + #face_img is bgr image + ret = self.detector.detect_face_limited(face_img, + det_type=self.args.det) + if ret is None: + return None + bbox, points = ret + if bbox.shape[0] == 0: + return None + bbox = bbox[0, 0:4] + points = points[0, :].reshape((2, 5)).T + #print(bbox) + #print(points) + nimg = face_preprocess.preprocess(face_img, + bbox, + points, + image_size='112,112') + nimg = cv2.cvtColor(nimg, cv2.COLOR_BGR2RGB) + aligned = np.transpose(nimg, (2, 0, 1)) + #print(nimg.shape) + embedding = None + for flipid in [0, 1]: + if flipid == 1: + if self.args.flip == 0: + break + do_flip(aligned) + input_blob = np.expand_dims(aligned, axis=0) + data = mx.nd.array(input_blob) + db = mx.io.DataBatch(data=(data, )) + self.model.forward(db, is_train=False) + _embedding = self.model.get_outputs()[0].asnumpy() + #print(_embedding.shape) + if embedding is None: + embedding = _embedding + else: + embedding += _embedding + embedding = sklearn.preprocessing.normalize(embedding).flatten() + return embedding diff --git a/deploy/face_model.py b/deploy/face_model.py index 6aee02f..42f472b 100644 --- a/deploy/face_model.py +++ b/deploy/face_model.py @@ -22,87 +22,98 @@ import face_preprocess def do_flip(data): - for idx in range(data.shape[0]): - data[idx,:,:] = np.fliplr(data[idx,:,:]) + for idx in range(data.shape[0]): + data[idx, :, :] = np.fliplr(data[idx, :, :]) + def get_model(ctx, image_size, model_str, layer): - _vec = model_str.split(',') - assert len(_vec)==2 - prefix = _vec[0] - epoch = int(_vec[1]) - print('loading',prefix, epoch) - sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) - all_layers = sym.get_internals() - sym = all_layers[layer+'_output'] - model = mx.mod.Module(symbol=sym, context=ctx, label_names = None) - #model.bind(data_shapes=[('data', (args.batch_size, 3, image_size[0], image_size[1]))], label_shapes=[('softmax_label', (args.batch_size,))]) - model.bind(data_shapes=[('data', (1, 3, image_size[0], image_size[1]))]) - model.set_params(arg_params, aux_params) - return model + _vec = model_str.split(',') + assert len(_vec) == 2 + prefix = _vec[0] + epoch = int(_vec[1]) + print('loading', prefix, epoch) + sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) + all_layers = sym.get_internals() + sym = all_layers[layer + '_output'] + model = mx.mod.Module(symbol=sym, context=ctx, label_names=None) + #model.bind(data_shapes=[('data', (args.batch_size, 3, image_size[0], image_size[1]))], label_shapes=[('softmax_label', (args.batch_size,))]) + model.bind(data_shapes=[('data', (1, 3, image_size[0], image_size[1]))]) + model.set_params(arg_params, aux_params) + return model + class FaceModel: - def __init__(self, args): - self.args = args - ctx = mx.gpu(args.gpu) - _vec = args.image_size.split(',') - assert len(_vec)==2 - image_size = (int(_vec[0]), int(_vec[1])) - self.model = None - self.ga_model = None - if len(args.model)>0: - self.model = get_model(ctx, image_size, args.model, 'fc1') - if len(args.ga_model)>0: - self.ga_model = get_model(ctx, image_size, args.ga_model, 'fc1') + def __init__(self, args): + self.args = args + ctx = mx.gpu(args.gpu) + _vec = args.image_size.split(',') + assert len(_vec) == 2 + image_size = (int(_vec[0]), int(_vec[1])) + self.model = None + self.ga_model = None + if len(args.model) > 0: + self.model = get_model(ctx, image_size, args.model, 'fc1') + if len(args.ga_model) > 0: + self.ga_model = get_model(ctx, image_size, args.ga_model, 'fc1') - self.threshold = args.threshold - self.det_minsize = 50 - self.det_threshold = [0.6,0.7,0.8] - #self.det_factor = 0.9 - self.image_size = image_size - mtcnn_path = os.path.join(os.path.dirname(__file__), 'mtcnn-model') - if args.det==0: - detector = MtcnnDetector(model_folder=mtcnn_path, ctx=ctx, num_worker=1, accurate_landmark = True, threshold=self.det_threshold) - else: - detector = MtcnnDetector(model_folder=mtcnn_path, ctx=ctx, num_worker=1, accurate_landmark = True, threshold=[0.0,0.0,0.2]) - self.detector = detector + self.threshold = args.threshold + self.det_minsize = 50 + self.det_threshold = [0.6, 0.7, 0.8] + #self.det_factor = 0.9 + self.image_size = image_size + mtcnn_path = os.path.join(os.path.dirname(__file__), 'mtcnn-model') + if args.det == 0: + detector = MtcnnDetector(model_folder=mtcnn_path, + ctx=ctx, + num_worker=1, + accurate_landmark=True, + threshold=self.det_threshold) + else: + detector = MtcnnDetector(model_folder=mtcnn_path, + ctx=ctx, + num_worker=1, + accurate_landmark=True, + threshold=[0.0, 0.0, 0.2]) + self.detector = detector + def get_input(self, face_img): + ret = self.detector.detect_face(face_img, det_type=self.args.det) + if ret is None: + return None + bbox, points = ret + if bbox.shape[0] == 0: + return None + bbox = bbox[0, 0:4] + points = points[0, :].reshape((2, 5)).T + #print(bbox) + #print(points) + nimg = face_preprocess.preprocess(face_img, + bbox, + points, + image_size='112,112') + nimg = cv2.cvtColor(nimg, cv2.COLOR_BGR2RGB) + aligned = np.transpose(nimg, (2, 0, 1)) + return aligned - def get_input(self, face_img): - ret = self.detector.detect_face(face_img, det_type = self.args.det) - if ret is None: - return None - bbox, points = ret - if bbox.shape[0]==0: - return None - bbox = bbox[0,0:4] - points = points[0,:].reshape((2,5)).T - #print(bbox) - #print(points) - nimg = face_preprocess.preprocess(face_img, bbox, points, image_size='112,112') - nimg = cv2.cvtColor(nimg, cv2.COLOR_BGR2RGB) - aligned = np.transpose(nimg, (2,0,1)) - return aligned + def get_feature(self, aligned): + input_blob = np.expand_dims(aligned, axis=0) + data = mx.nd.array(input_blob) + db = mx.io.DataBatch(data=(data, )) + self.model.forward(db, is_train=False) + embedding = self.model.get_outputs()[0].asnumpy() + embedding = sklearn.preprocessing.normalize(embedding).flatten() + return embedding - def get_feature(self, aligned): - input_blob = np.expand_dims(aligned, axis=0) - data = mx.nd.array(input_blob) - db = mx.io.DataBatch(data=(data,)) - self.model.forward(db, is_train=False) - embedding = self.model.get_outputs()[0].asnumpy() - embedding = sklearn.preprocessing.normalize(embedding).flatten() - return embedding - - def get_ga(self, aligned): - input_blob = np.expand_dims(aligned, axis=0) - data = mx.nd.array(input_blob) - db = mx.io.DataBatch(data=(data,)) - self.ga_model.forward(db, is_train=False) - ret = self.ga_model.get_outputs()[0].asnumpy() - g = ret[:,0:2].flatten() - gender = np.argmax(g) - a = ret[:,2:202].reshape( (100,2) ) - a = np.argmax(a, axis=1) - age = int(sum(a)) - - return gender, age + def get_ga(self, aligned): + input_blob = np.expand_dims(aligned, axis=0) + data = mx.nd.array(input_blob) + db = mx.io.DataBatch(data=(data, )) + self.ga_model.forward(db, is_train=False) + ret = self.ga_model.get_outputs()[0].asnumpy() + g = ret[:, 0:2].flatten() + gender = np.argmax(g) + a = ret[:, 2:202].reshape((100, 2)) + a = np.argmax(a, axis=1) + age = int(sum(a)) + return gender, age diff --git a/deploy/ga_merge.py b/deploy/ga_merge.py index d6397f0..36f512f 100644 --- a/deploy/ga_merge.py +++ b/deploy/ga_merge.py @@ -11,7 +11,9 @@ import mxnet as mx parser = argparse.ArgumentParser(description='merge age and gender models') # general parser.add_argument('--age-model', default='', help='path to load age model.') -parser.add_argument('--gender-model', default='', help='path to load gender model.') +parser.add_argument('--gender-model', + default='', + help='path to load gender model.') parser.add_argument('--prefix', default='', help='path to save model.') args = parser.parse_args() @@ -20,33 +22,32 @@ tsym = None targ = {} taux = {} for model in [args.age_model, args.gender_model]: - _vec = model.split(',') - assert len(_vec)==2 - prefix = _vec[0] - epoch = int(_vec[1]) - print('loading',prefix, epoch) - sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) - if tsym is None: - all_layers = sym.get_internals() - tsym = all_layers['fc1_output'] - if i==0: - prefix = 'age' - else: - prefix = 'gender' - for k,v in arg_params.iteritems(): - if k.startswith(prefix): - print('arg', i, k) - targ[k] = v - for k,v in aux_params.iteritems(): - if k.startswith(prefix): - print('aux', i, k) - taux[k] = v - i+=1 + _vec = model.split(',') + assert len(_vec) == 2 + prefix = _vec[0] + epoch = int(_vec[1]) + print('loading', prefix, epoch) + sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) + if tsym is None: + all_layers = sym.get_internals() + tsym = all_layers['fc1_output'] + if i == 0: + prefix = 'age' + else: + prefix = 'gender' + for k, v in arg_params.iteritems(): + if k.startswith(prefix): + print('arg', i, k) + targ[k] = v + for k, v in aux_params.iteritems(): + if k.startswith(prefix): + print('aux', i, k) + taux[k] = v + i += 1 dellist = [] #for k,v in arg_params.iteritems(): # if k.startswith('fc7'): # dellist.append(k) for d in dellist: - del targ[d] + del targ[d] mx.model.save_checkpoint(args.prefix, 0, tsym, targ, taux) - diff --git a/deploy/helper.py b/deploy/helper.py index b82c4b7..38f2c9c 100644 --- a/deploy/helper.py +++ b/deploy/helper.py @@ -61,11 +61,13 @@ def nms(boxes, overlap_threshold, mode='Union'): overlap = inter / (area[i] + area[idxs[:last]] - inter) # delete all indexes from the index list that have - idxs = np.delete(idxs, np.concatenate(([last], - np.where(overlap > overlap_threshold)[0]))) + idxs = np.delete( + idxs, + np.concatenate(([last], np.where(overlap > overlap_threshold)[0]))) return pick + def adjust_input(in_data): """ adjust the input from (h, w, c) to ( 1, c, h, w) for network input @@ -84,13 +86,14 @@ def adjust_input(in_data): else: out_data = in_data - out_data = out_data.transpose((2,0,1)) + out_data = out_data.transpose((2, 0, 1)) out_data = np.expand_dims(out_data, 0) - out_data = (out_data - 127.5)*0.0078125 + out_data = (out_data - 127.5) * 0.0078125 return out_data + def generate_bbox(map, reg, scale, threshold): - """ + """ generate bbox from feature map Parameters: ---------- @@ -106,27 +109,27 @@ def generate_bbox(map, reg, scale, threshold): ------- bbox array """ - stride = 2 - cellsize = 12 + stride = 2 + cellsize = 12 - t_index = np.where(map>threshold) + t_index = np.where(map > threshold) - # find nothing - if t_index[0].size == 0: - return np.array([]) + # find nothing + if t_index[0].size == 0: + return np.array([]) - dx1, dy1, dx2, dy2 = [reg[0, i, t_index[0], t_index[1]] for i in range(4)] + dx1, dy1, dx2, dy2 = [reg[0, i, t_index[0], t_index[1]] for i in range(4)] - reg = np.array([dx1, dy1, dx2, dy2]) - score = map[t_index[0], t_index[1]] - boundingbox = np.vstack([np.round((stride*t_index[1]+1)/scale), - np.round((stride*t_index[0]+1)/scale), - np.round((stride*t_index[1]+1+cellsize)/scale), - np.round((stride*t_index[0]+1+cellsize)/scale), - score, - reg]) + reg = np.array([dx1, dy1, dx2, dy2]) + score = map[t_index[0], t_index[1]] + boundingbox = np.vstack([ + np.round((stride * t_index[1] + 1) / scale), + np.round((stride * t_index[0] + 1) / scale), + np.round((stride * t_index[1] + 1 + cellsize) / scale), + np.round((stride * t_index[0] + 1 + cellsize) / scale), score, reg + ]) - return boundingbox.T + return boundingbox.T def detect_first_stage(img, net, scale, threshold): @@ -148,21 +151,22 @@ def detect_first_stage(img, net, scale, threshold): height, width, _ = img.shape hs = int(math.ceil(height * scale)) ws = int(math.ceil(width * scale)) - - im_data = cv2.resize(img, (ws,hs)) - + + im_data = cv2.resize(img, (ws, hs)) + # adjust for the network input input_buf = adjust_input(im_data) output = net.predict(input_buf) - boxes = generate_bbox(output[1][0,1,:,:], output[0], scale, threshold) + boxes = generate_bbox(output[1][0, 1, :, :], output[0], scale, threshold) if boxes.size == 0: return None # nms - pick = nms(boxes[:,0:5], 0.5, mode='Union') + pick = nms(boxes[:, 0:5], 0.5, mode='Union') boxes = boxes[pick] return boxes -def detect_first_stage_warpper( args ): + +def detect_first_stage_warpper(args): return detect_first_stage(*args) diff --git a/deploy/model_slim.py b/deploy/model_slim.py index 327adde..421b0cd 100644 --- a/deploy/model_slim.py +++ b/deploy/model_slim.py @@ -10,22 +10,23 @@ import mxnet as mx parser = argparse.ArgumentParser(description='face model slim') # general -parser.add_argument('--model', default='../models/model-r34-amf/model,60', help='path to load model.') +parser.add_argument('--model', + default='../models/model-r34-amf/model,60', + help='path to load model.') args = parser.parse_args() _vec = args.model.split(',') -assert len(_vec)==2 +assert len(_vec) == 2 prefix = _vec[0] epoch = int(_vec[1]) -print('loading',prefix, epoch) +print('loading', prefix, epoch) sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) all_layers = sym.get_internals() sym = all_layers['fc1_output'] dellist = [] -for k,v in arg_params.iteritems(): - if k.startswith('fc7'): - dellist.append(k) +for k, v in arg_params.iteritems(): + if k.startswith('fc7'): + dellist.append(k) for d in dellist: - del arg_params[d] -mx.model.save_checkpoint(prefix+"s", 0, sym, arg_params, aux_params) - + del arg_params[d] +mx.model.save_checkpoint(prefix + "s", 0, sym, arg_params, aux_params) diff --git a/deploy/mtcnn_detector.py b/deploy/mtcnn_detector.py index c7332a5..8d9f0fe 100644 --- a/deploy/mtcnn_detector.py +++ b/deploy/mtcnn_detector.py @@ -13,6 +13,7 @@ except ImportError: from helper import nms, adjust_input, generate_bbox, detect_first_stage_warpper + class MtcnnDetector(object): """ Joint Face Detection and Alignment using Multi-task Cascaded Convolutional Neural Networks @@ -21,11 +22,11 @@ class MtcnnDetector(object): """ def __init__(self, model_folder='.', - minsize = 20, - threshold = [0.6, 0.7, 0.8], - factor = 0.709, - num_worker = 1, - accurate_landmark = False, + minsize=20, + threshold=[0.6, 0.7, 0.8], + factor=0.709, + num_worker=1, + accurate_landmark=False, ctx=mx.cpu()): """ Initialize the detector @@ -50,9 +51,9 @@ class MtcnnDetector(object): self.accurate_landmark = accurate_landmark # load 4 models from folder - models = ['det1', 'det2', 'det3','det4'] - models = [ os.path.join(model_folder, f) for f in models] - + models = ['det1', 'det2', 'det3', 'det4'] + models = [os.path.join(model_folder, f) for f in models] + self.PNets = [] for i in range(num_worker): workner_net = mx.model.FeedForward.load(models[0], 1, ctx=ctx) @@ -64,11 +65,10 @@ class MtcnnDetector(object): self.ONet = mx.model.FeedForward.load(models[2], 1, ctx=ctx) self.LNet = mx.model.FeedForward.load(models[3], 1, ctx=ctx) - self.minsize = float(minsize) - self.factor = float(factor) + self.minsize = float(minsize) + self.factor = float(factor) self.threshold = threshold - def convert_to_square(self, bbox): """ convert bbox to square @@ -86,9 +86,9 @@ class MtcnnDetector(object): h = bbox[:, 3] - bbox[:, 1] + 1 w = bbox[:, 2] - bbox[:, 0] + 1 - max_side = np.maximum(h,w) - square_bbox[:, 0] = bbox[:, 0] + w*0.5 - max_side*0.5 - square_bbox[:, 1] = bbox[:, 1] + h*0.5 - max_side*0.5 + max_side = np.maximum(h, w) + square_bbox[:, 0] = bbox[:, 0] + w * 0.5 - max_side * 0.5 + square_bbox[:, 1] = bbox[:, 1] + h * 0.5 - max_side * 0.5 square_bbox[:, 2] = square_bbox[:, 0] + max_side - 1 square_bbox[:, 3] = square_bbox[:, 1] + max_side - 1 return square_bbox @@ -118,7 +118,6 @@ class MtcnnDetector(object): bbox[:, 0:4] = bbox[:, 0:4] + aug return bbox - def pad(self, bboxes, w, h): """ pad the the bboxes, alse restrict the size of it @@ -145,19 +144,21 @@ class MtcnnDetector(object): height and width of the bbox """ - tmpw, tmph = bboxes[:, 2] - bboxes[:, 0] + 1, bboxes[:, 3] - bboxes[:, 1] + 1 + tmpw, tmph = bboxes[:, 2] - bboxes[:, 0] + 1, bboxes[:, + 3] - bboxes[:, + 1] + 1 num_box = bboxes.shape[0] - dx , dy= np.zeros((num_box, )), np.zeros((num_box, )) - edx, edy = tmpw.copy()-1, tmph.copy()-1 + dx, dy = np.zeros((num_box, )), np.zeros((num_box, )) + edx, edy = tmpw.copy() - 1, tmph.copy() - 1 x, y, ex, ey = bboxes[:, 0], bboxes[:, 1], bboxes[:, 2], bboxes[:, 3] - tmp_index = np.where(ex > w-1) + tmp_index = np.where(ex > w - 1) edx[tmp_index] = tmpw[tmp_index] + w - 2 - ex[tmp_index] ex[tmp_index] = w - 1 - tmp_index = np.where(ey > h-1) + tmp_index = np.where(ey > h - 1) edy[tmp_index] = tmph[tmp_index] + h - 2 - ey[tmp_index] ey[tmp_index] = h - 1 @@ -172,7 +173,7 @@ class MtcnnDetector(object): return_list = [dy, edy, dx, edx, y, ey, x, ex, tmpw, tmph] return_list = [item.astype(np.int32) for item in return_list] - return return_list + return return_list def slice_index(self, number): """ @@ -186,53 +187,63 @@ class MtcnnDetector(object): """Yield successive n-sized chunks from l.""" for i in range(0, len(l), n): yield l[i:i + n] + num_list = range(number) return list(chunks(num_list, self.num_worker)) - + def detect_face_limited(self, img, det_type=2): height, width, _ = img.shape - if det_type>=2: - total_boxes = np.array( [ [0.0, 0.0, img.shape[1], img.shape[0], 0.9] ] ,dtype=np.float32) - num_box = total_boxes.shape[0] + if det_type >= 2: + total_boxes = np.array( + [[0.0, 0.0, img.shape[1], img.shape[0], 0.9]], + dtype=np.float32) + num_box = total_boxes.shape[0] - # pad the bbox - [dy, edy, dx, edx, y, ey, x, ex, tmpw, tmph] = self.pad(total_boxes, width, height) - # (3, 24, 24) is the input shape for RNet - input_buf = np.zeros((num_box, 3, 24, 24), dtype=np.float32) + # pad the bbox + [dy, edy, dx, edx, y, ey, x, ex, tmpw, + tmph] = self.pad(total_boxes, width, height) + # (3, 24, 24) is the input shape for RNet + input_buf = np.zeros((num_box, 3, 24, 24), dtype=np.float32) - for i in range(num_box): - tmp = np.zeros((tmph[i], tmpw[i], 3), dtype=np.uint8) - tmp[dy[i]:edy[i]+1, dx[i]:edx[i]+1, :] = img[y[i]:ey[i]+1, x[i]:ex[i]+1, :] - input_buf[i, :, :, :] = adjust_input(cv2.resize(tmp, (24, 24))) + for i in range(num_box): + tmp = np.zeros((tmph[i], tmpw[i], 3), dtype=np.uint8) + tmp[dy[i]:edy[i] + 1, + dx[i]:edx[i] + 1, :] = img[y[i]:ey[i] + 1, + x[i]:ex[i] + 1, :] + input_buf[i, :, :, :] = adjust_input(cv2.resize(tmp, (24, 24))) - output = self.RNet.predict(input_buf) + output = self.RNet.predict(input_buf) - # filter the total_boxes with threshold - passed = np.where(output[1][:, 1] > self.threshold[1]) - total_boxes = total_boxes[passed] + # filter the total_boxes with threshold + passed = np.where(output[1][:, 1] > self.threshold[1]) + total_boxes = total_boxes[passed] - if total_boxes.size == 0: - return None + if total_boxes.size == 0: + return None - total_boxes[:, 4] = output[1][passed, 1].reshape((-1,)) - reg = output[0][passed] + total_boxes[:, 4] = output[1][passed, 1].reshape((-1, )) + reg = output[0][passed] - # nms - pick = nms(total_boxes, 0.7, 'Union') - total_boxes = total_boxes[pick] - total_boxes = self.calibrate_box(total_boxes, reg[pick]) - total_boxes = self.convert_to_square(total_boxes) - total_boxes[:, 0:4] = np.round(total_boxes[:, 0:4]) + # nms + pick = nms(total_boxes, 0.7, 'Union') + total_boxes = total_boxes[pick] + total_boxes = self.calibrate_box(total_boxes, reg[pick]) + total_boxes = self.convert_to_square(total_boxes) + total_boxes[:, 0:4] = np.round(total_boxes[:, 0:4]) else: - total_boxes = np.array( [ [0.0, 0.0, img.shape[1], img.shape[0], 0.9] ] ,dtype=np.float32) + total_boxes = np.array( + [[0.0, 0.0, img.shape[1], img.shape[0], 0.9]], + dtype=np.float32) num_box = total_boxes.shape[0] - [dy, edy, dx, edx, y, ey, x, ex, tmpw, tmph] = self.pad(total_boxes, width, height) + [dy, edy, dx, edx, y, ey, x, ex, tmpw, + tmph] = self.pad(total_boxes, width, height) # (3, 48, 48) is the input shape for ONet input_buf = np.zeros((num_box, 3, 48, 48), dtype=np.float32) for i in range(num_box): tmp = np.zeros((tmph[i], tmpw[i], 3), dtype=np.float32) - tmp[dy[i]:edy[i]+1, dx[i]:edx[i]+1, :] = img[y[i]:ey[i]+1, x[i]:ex[i]+1, :] + tmp[dy[i]:edy[i] + 1, dx[i]:edx[i] + 1, :] = img[y[i]:ey[i] + 1, + x[i]:ex[i] + 1, :] input_buf[i, :, :, :] = adjust_input(cv2.resize(tmp, (48, 48))) output = self.ONet.predict(input_buf) @@ -245,22 +256,24 @@ class MtcnnDetector(object): if total_boxes.size == 0: return None - total_boxes[:, 4] = output[2][passed, 1].reshape((-1,)) + total_boxes[:, 4] = output[2][passed, 1].reshape((-1, )) reg = output[1][passed] points = output[0][passed] # compute landmark points bbw = total_boxes[:, 2] - total_boxes[:, 0] + 1 bbh = total_boxes[:, 3] - total_boxes[:, 1] + 1 - points[:, 0:5] = np.expand_dims(total_boxes[:, 0], 1) + np.expand_dims(bbw, 1) * points[:, 0:5] - points[:, 5:10] = np.expand_dims(total_boxes[:, 1], 1) + np.expand_dims(bbh, 1) * points[:, 5:10] + points[:, 0:5] = np.expand_dims( + total_boxes[:, 0], 1) + np.expand_dims(bbw, 1) * points[:, 0:5] + points[:, 5:10] = np.expand_dims( + total_boxes[:, 1], 1) + np.expand_dims(bbh, 1) * points[:, 5:10] # nms total_boxes = self.calibrate_box(total_boxes, reg) pick = nms(total_boxes, 0.7, 'Min') total_boxes = total_boxes[pick] points = points[pick] - + if not self.accurate_landmark: return total_boxes, points @@ -268,23 +281,27 @@ class MtcnnDetector(object): # extended stage ############################################# num_box = total_boxes.shape[0] - patchw = np.maximum(total_boxes[:, 2]-total_boxes[:, 0]+1, total_boxes[:, 3]-total_boxes[:, 1]+1) - patchw = np.round(patchw*0.25) + patchw = np.maximum(total_boxes[:, 2] - total_boxes[:, 0] + 1, + total_boxes[:, 3] - total_boxes[:, 1] + 1) + patchw = np.round(patchw * 0.25) # make it even - patchw[np.where(np.mod(patchw,2) == 1)] += 1 + patchw[np.where(np.mod(patchw, 2) == 1)] += 1 input_buf = np.zeros((num_box, 15, 24, 24), dtype=np.float32) for i in range(5): - x, y = points[:, i], points[:, i+5] - x, y = np.round(x-0.5*patchw), np.round(y-0.5*patchw) - [dy, edy, dx, edx, y, ey, x, ex, tmpw, tmph] = self.pad(np.vstack([x, y, x+patchw-1, y+patchw-1]).T, - width, - height) + x, y = points[:, i], points[:, i + 5] + x, y = np.round(x - 0.5 * patchw), np.round(y - 0.5 * patchw) + [dy, edy, dx, edx, y, ey, x, ex, tmpw, tmph] = self.pad( + np.vstack([x, y, x + patchw - 1, y + patchw - 1]).T, width, + height) for j in range(num_box): tmpim = np.zeros((tmpw[j], tmpw[j], 3), dtype=np.float32) - tmpim[dy[j]:edy[j]+1, dx[j]:edx[j]+1, :] = img[y[j]:ey[j]+1, x[j]:ex[j]+1, :] - input_buf[j, i*3:i*3+3, :, :] = adjust_input(cv2.resize(tmpim, (24, 24))) + tmpim[dy[j]:edy[j] + 1, + dx[j]:edx[j] + 1, :] = img[y[j]:ey[j] + 1, + x[j]:ex[j] + 1, :] + input_buf[j, i * 3:i * 3 + 3, :, :] = adjust_input( + cv2.resize(tmpim, (24, 24))) output = self.LNet.predict(input_buf) @@ -293,11 +310,13 @@ class MtcnnDetector(object): for k in range(5): # do not make a large movement - tmp_index = np.where(np.abs(output[k]-0.5) > 0.35) + tmp_index = np.where(np.abs(output[k] - 0.5) > 0.35) output[k][tmp_index[0]] = 0.5 - pointx[:, k] = np.round(points[:, k] - 0.5*patchw) + output[k][:, 0]*patchw - pointy[:, k] = np.round(points[:, k+5] - 0.5*patchw) + output[k][:, 1]*patchw + pointx[:, k] = np.round(points[:, k] - + 0.5 * patchw) + output[k][:, 0] * patchw + pointy[:, k] = np.round(points[:, k + 5] - + 0.5 * patchw) + output[k][:, 1] * patchw points = np.hstack([pointx, pointy]) points = points.astype(np.int32) @@ -321,7 +340,7 @@ class MtcnnDetector(object): # check input height, width, _ = img.shape - if det_type==0: + if det_type == 0: MIN_DET_SIZE = 12 if img is None: @@ -334,15 +353,15 @@ class MtcnnDetector(object): # detected boxes total_boxes = [] - minl = min( height, width) + minl = min(height, width) # get all the valid scales scales = [] - m = MIN_DET_SIZE/self.minsize + m = MIN_DET_SIZE / self.minsize minl *= m factor_count = 0 while minl > MIN_DET_SIZE: - scales.append(m*self.factor**factor_count) + scales.append(m * self.factor**factor_count) minl *= self.factor factor_count += 1 @@ -353,7 +372,7 @@ class MtcnnDetector(object): # return_boxes = self.detect_first_stage(img, scale, 0) # if return_boxes is not None: # total_boxes.append(return_boxes) - + sliced_index = self.slice_index(len(scales)) total_boxes = [] for batch in sliced_index: @@ -362,13 +381,13 @@ class MtcnnDetector(object): local_boxes = map( detect_first_stage_warpper, \ izip(repeat(img), self.PNets[:len(batch)], [scales[i] for i in batch], repeat(self.threshold[0])) ) total_boxes.extend(local_boxes) - - # remove the Nones - total_boxes = [ i for i in total_boxes if i is not None] + + # remove the Nones + total_boxes = [i for i in total_boxes if i is not None] if len(total_boxes) == 0: return None - + total_boxes = np.vstack(total_boxes) if total_boxes.size == 0: @@ -382,18 +401,20 @@ class MtcnnDetector(object): bbh = total_boxes[:, 3] - total_boxes[:, 1] + 1 # refine the bboxes - total_boxes = np.vstack([total_boxes[:, 0]+total_boxes[:, 5] * bbw, - total_boxes[:, 1]+total_boxes[:, 6] * bbh, - total_boxes[:, 2]+total_boxes[:, 7] * bbw, - total_boxes[:, 3]+total_boxes[:, 8] * bbh, - total_boxes[:, 4] - ]) + total_boxes = np.vstack([ + total_boxes[:, 0] + total_boxes[:, 5] * bbw, + total_boxes[:, 1] + total_boxes[:, 6] * bbh, + total_boxes[:, 2] + total_boxes[:, 7] * bbw, + total_boxes[:, 3] + total_boxes[:, 8] * bbh, total_boxes[:, 4] + ]) total_boxes = total_boxes.T total_boxes = self.convert_to_square(total_boxes) total_boxes[:, 0:4] = np.round(total_boxes[:, 0:4]) else: - total_boxes = np.array( [ [0.0, 0.0, img.shape[1], img.shape[0], 0.9] ] ,dtype=np.float32) + total_boxes = np.array( + [[0.0, 0.0, img.shape[1], img.shape[0], 0.9]], + dtype=np.float32) ############################################# # second stage @@ -401,13 +422,15 @@ class MtcnnDetector(object): num_box = total_boxes.shape[0] # pad the bbox - [dy, edy, dx, edx, y, ey, x, ex, tmpw, tmph] = self.pad(total_boxes, width, height) + [dy, edy, dx, edx, y, ey, x, ex, tmpw, + tmph] = self.pad(total_boxes, width, height) # (3, 24, 24) is the input shape for RNet input_buf = np.zeros((num_box, 3, 24, 24), dtype=np.float32) for i in range(num_box): tmp = np.zeros((tmph[i], tmpw[i], 3), dtype=np.uint8) - tmp[dy[i]:edy[i]+1, dx[i]:edx[i]+1, :] = img[y[i]:ey[i]+1, x[i]:ex[i]+1, :] + tmp[dy[i]:edy[i] + 1, dx[i]:edx[i] + 1, :] = img[y[i]:ey[i] + 1, + x[i]:ex[i] + 1, :] input_buf[i, :, :, :] = adjust_input(cv2.resize(tmp, (24, 24))) output = self.RNet.predict(input_buf) @@ -419,7 +442,7 @@ class MtcnnDetector(object): if total_boxes.size == 0: return None - total_boxes[:, 4] = output[1][passed, 1].reshape((-1,)) + total_boxes[:, 4] = output[1][passed, 1].reshape((-1, )) reg = output[0][passed] # nms @@ -435,13 +458,15 @@ class MtcnnDetector(object): num_box = total_boxes.shape[0] # pad the bbox - [dy, edy, dx, edx, y, ey, x, ex, tmpw, tmph] = self.pad(total_boxes, width, height) + [dy, edy, dx, edx, y, ey, x, ex, tmpw, + tmph] = self.pad(total_boxes, width, height) # (3, 48, 48) is the input shape for ONet input_buf = np.zeros((num_box, 3, 48, 48), dtype=np.float32) for i in range(num_box): tmp = np.zeros((tmph[i], tmpw[i], 3), dtype=np.float32) - tmp[dy[i]:edy[i]+1, dx[i]:edx[i]+1, :] = img[y[i]:ey[i]+1, x[i]:ex[i]+1, :] + tmp[dy[i]:edy[i] + 1, dx[i]:edx[i] + 1, :] = img[y[i]:ey[i] + 1, + x[i]:ex[i] + 1, :] input_buf[i, :, :, :] = adjust_input(cv2.resize(tmp, (48, 48))) output = self.ONet.predict(input_buf) @@ -453,22 +478,24 @@ class MtcnnDetector(object): if total_boxes.size == 0: return None - total_boxes[:, 4] = output[2][passed, 1].reshape((-1,)) + total_boxes[:, 4] = output[2][passed, 1].reshape((-1, )) reg = output[1][passed] points = output[0][passed] # compute landmark points bbw = total_boxes[:, 2] - total_boxes[:, 0] + 1 bbh = total_boxes[:, 3] - total_boxes[:, 1] + 1 - points[:, 0:5] = np.expand_dims(total_boxes[:, 0], 1) + np.expand_dims(bbw, 1) * points[:, 0:5] - points[:, 5:10] = np.expand_dims(total_boxes[:, 1], 1) + np.expand_dims(bbh, 1) * points[:, 5:10] + points[:, 0:5] = np.expand_dims( + total_boxes[:, 0], 1) + np.expand_dims(bbw, 1) * points[:, 0:5] + points[:, 5:10] = np.expand_dims( + total_boxes[:, 1], 1) + np.expand_dims(bbh, 1) * points[:, 5:10] # nms total_boxes = self.calibrate_box(total_boxes, reg) pick = nms(total_boxes, 0.7, 'Min') total_boxes = total_boxes[pick] points = points[pick] - + if not self.accurate_landmark: return total_boxes, points @@ -476,23 +503,27 @@ class MtcnnDetector(object): # extended stage ############################################# num_box = total_boxes.shape[0] - patchw = np.maximum(total_boxes[:, 2]-total_boxes[:, 0]+1, total_boxes[:, 3]-total_boxes[:, 1]+1) - patchw = np.round(patchw*0.25) + patchw = np.maximum(total_boxes[:, 2] - total_boxes[:, 0] + 1, + total_boxes[:, 3] - total_boxes[:, 1] + 1) + patchw = np.round(patchw * 0.25) # make it even - patchw[np.where(np.mod(patchw,2) == 1)] += 1 + patchw[np.where(np.mod(patchw, 2) == 1)] += 1 input_buf = np.zeros((num_box, 15, 24, 24), dtype=np.float32) for i in range(5): - x, y = points[:, i], points[:, i+5] - x, y = np.round(x-0.5*patchw), np.round(y-0.5*patchw) - [dy, edy, dx, edx, y, ey, x, ex, tmpw, tmph] = self.pad(np.vstack([x, y, x+patchw-1, y+patchw-1]).T, - width, - height) + x, y = points[:, i], points[:, i + 5] + x, y = np.round(x - 0.5 * patchw), np.round(y - 0.5 * patchw) + [dy, edy, dx, edx, y, ey, x, ex, tmpw, tmph] = self.pad( + np.vstack([x, y, x + patchw - 1, y + patchw - 1]).T, width, + height) for j in range(num_box): tmpim = np.zeros((tmpw[j], tmpw[j], 3), dtype=np.float32) - tmpim[dy[j]:edy[j]+1, dx[j]:edx[j]+1, :] = img[y[j]:ey[j]+1, x[j]:ex[j]+1, :] - input_buf[j, i*3:i*3+3, :, :] = adjust_input(cv2.resize(tmpim, (24, 24))) + tmpim[dy[j]:edy[j] + 1, + dx[j]:edx[j] + 1, :] = img[y[j]:ey[j] + 1, + x[j]:ex[j] + 1, :] + input_buf[j, i * 3:i * 3 + 3, :, :] = adjust_input( + cv2.resize(tmpim, (24, 24))) output = self.LNet.predict(input_buf) @@ -501,19 +532,19 @@ class MtcnnDetector(object): for k in range(5): # do not make a large movement - tmp_index = np.where(np.abs(output[k]-0.5) > 0.35) + tmp_index = np.where(np.abs(output[k] - 0.5) > 0.35) output[k][tmp_index[0]] = 0.5 - pointx[:, k] = np.round(points[:, k] - 0.5*patchw) + output[k][:, 0]*patchw - pointy[:, k] = np.round(points[:, k+5] - 0.5*patchw) + output[k][:, 1]*patchw + pointx[:, k] = np.round(points[:, k] - + 0.5 * patchw) + output[k][:, 0] * patchw + pointy[:, k] = np.round(points[:, k + 5] - + 0.5 * patchw) + output[k][:, 1] * patchw points = np.hstack([pointx, pointy]) points = points.astype(np.int32) return total_boxes, points - - def list2colmatrix(self, pts_list): """ convert list to column matrix @@ -546,15 +577,16 @@ class MtcnnDetector(object): tran_m: tran_b: """ - assert from_shape.shape[0] == to_shape.shape[0] and from_shape.shape[0] % 2 == 0 + assert from_shape.shape[0] == to_shape.shape[ + 0] and from_shape.shape[0] % 2 == 0 sigma_from = 0.0 sigma_to = 0.0 cov = np.matrix([[0.0, 0.0], [0.0, 0.0]]) # compute the mean and cov - from_shape_points = from_shape.reshape(from_shape.shape[0]/2, 2) - to_shape_points = to_shape.reshape(to_shape.shape[0]/2, 2) + from_shape_points = from_shape.reshape(from_shape.shape[0] / 2, 2) + to_shape_points = to_shape.reshape(to_shape.shape[0] / 2, 2) mean_from = from_shape_points.mean(axis=0) mean_to = to_shape_points.mean(axis=0) @@ -563,7 +595,8 @@ class MtcnnDetector(object): sigma_from += temp_dis * temp_dis temp_dis = np.linalg.norm(to_shape_points[i] - mean_to) sigma_to += temp_dis * temp_dis - cov += (to_shape_points[i].transpose() - mean_to.transpose()) * (from_shape_points[i] - mean_from) + cov += (to_shape_points[i].transpose() - + mean_to.transpose()) * (from_shape_points[i] - mean_from) sigma_from = sigma_from / to_shape_points.shape[0] sigma_to = sigma_to / to_shape_points.shape[0] @@ -605,27 +638,33 @@ class MtcnnDetector(object): """ crop_imgs = [] for p in points: - shape =[] - for k in range(len(p)/2): + shape = [] + for k in range(len(p) / 2): shape.append(p[k]) - shape.append(p[k+5]) + shape.append(p[k + 5]) if padding > 0: padding = padding else: padding = 0 # average positions of face points - mean_face_shape_x = [0.224152, 0.75610125, 0.490127, 0.254149, 0.726104] - mean_face_shape_y = [0.2119465, 0.2119465, 0.628106, 0.780233, 0.780233] + mean_face_shape_x = [ + 0.224152, 0.75610125, 0.490127, 0.254149, 0.726104 + ] + mean_face_shape_y = [ + 0.2119465, 0.2119465, 0.628106, 0.780233, 0.780233 + ] from_points = [] to_points = [] - for i in range(len(shape)/2): - x = (padding + mean_face_shape_x[i]) / (2 * padding + 1) * desired_size - y = (padding + mean_face_shape_y[i]) / (2 * padding + 1) * desired_size + for i in range(len(shape) / 2): + x = (padding + mean_face_shape_x[i]) / (2 * padding + + 1) * desired_size + y = (padding + mean_face_shape_y[i]) / (2 * padding + + 1) * desired_size to_points.append([x, y]) - from_points.append([shape[2*i], shape[2*i+1]]) + from_points.append([shape[2 * i], shape[2 * i + 1]]) # convert the points to Mat from_mat = self.list2colmatrix(from_points) @@ -638,9 +677,11 @@ class MtcnnDetector(object): probe_vec = tran_m * probe_vec scale = np.linalg.norm(probe_vec) - angle = 180.0 / math.pi * math.atan2(probe_vec[1, 0], probe_vec[0, 0]) + angle = 180.0 / math.pi * math.atan2(probe_vec[1, 0], probe_vec[0, + 0]) - from_center = [(shape[0]+shape[2])/2.0, (shape[1]+shape[3])/2.0] + from_center = [(shape[0] + shape[2]) / 2.0, + (shape[1] + shape[3]) / 2.0] to_center = [0, 0] to_center[1] = desired_size * 0.4 to_center[0] = desired_size * 0.5 @@ -648,7 +689,8 @@ class MtcnnDetector(object): ex = to_center[0] - from_center[0] ey = to_center[1] - from_center[1] - rot_mat = cv2.getRotationMatrix2D((from_center[0], from_center[1]), -1*angle, scale) + rot_mat = cv2.getRotationMatrix2D((from_center[0], from_center[1]), + -1 * angle, scale) rot_mat[0][2] += ex rot_mat[1][2] += ey @@ -656,4 +698,3 @@ class MtcnnDetector(object): crop_imgs.append(chips) return crop_imgs - diff --git a/deploy/test.py b/deploy/test.py index 6150a08..fa9e611 100644 --- a/deploy/test.py +++ b/deploy/test.py @@ -10,9 +10,19 @@ parser.add_argument('--image-size', default='112,112', help='') parser.add_argument('--model', default='', help='path to load model.') parser.add_argument('--ga-model', default='', help='path to load model.') parser.add_argument('--gpu', default=0, type=int, help='gpu id') -parser.add_argument('--det', default=0, type=int, help='mtcnn option, 1 means using R+O, 0 means detect from begining') -parser.add_argument('--flip', default=0, type=int, help='whether do lr flip aug') -parser.add_argument('--threshold', default=1.24, type=float, help='ver dist threshold') +parser.add_argument( + '--det', + default=0, + type=int, + help='mtcnn option, 1 means using R+O, 0 means detect from begining') +parser.add_argument('--flip', + default=0, + type=int, + help='whether do lr flip aug') +parser.add_argument('--threshold', + default=1.24, + type=float, + help='ver dist threshold') args = parser.parse_args() model = face_model.FaceModel(args) @@ -24,9 +34,11 @@ gender, age = model.get_ga(img) print(gender) print(age) sys.exit(0) -img = cv2.imread('/raid5data/dplearn/megaface/facescrubr/112x112/Tom_Hanks/Tom_Hanks_54733.png') +img = cv2.imread( + '/raid5data/dplearn/megaface/facescrubr/112x112/Tom_Hanks/Tom_Hanks_54733.png' +) f2 = model.get_feature(img) -dist = np.sum(np.square(f1-f2)) +dist = np.sum(np.square(f1 - f2)) print(dist) sim = np.dot(f1, f2.T) print(sim) diff --git a/RetinaFace/Makefile b/detection/RetinaFace/Makefile similarity index 100% rename from RetinaFace/Makefile rename to detection/RetinaFace/Makefile diff --git a/RetinaFace/README.md b/detection/RetinaFace/README.md similarity index 100% rename from RetinaFace/README.md rename to detection/RetinaFace/README.md diff --git a/RetinaFace/rcnn/PY_OP/__init__.py b/detection/RetinaFace/rcnn/PY_OP/__init__.py similarity index 100% rename from RetinaFace/rcnn/PY_OP/__init__.py rename to detection/RetinaFace/rcnn/PY_OP/__init__.py diff --git a/detection/RetinaFace/rcnn/PY_OP/cascade_refine.py b/detection/RetinaFace/rcnn/PY_OP/cascade_refine.py new file mode 100644 index 0000000..e3c6556 --- /dev/null +++ b/detection/RetinaFace/rcnn/PY_OP/cascade_refine.py @@ -0,0 +1,518 @@ +from __future__ import print_function +import sys +import mxnet as mx +import numpy as np +import datetime +from distutils.util import strtobool +from ..config import config, generate_config +from ..processing.generate_anchor import generate_anchors, anchors_plane +from ..processing.bbox_transform import bbox_overlaps, bbox_transform, landmark_transform + +STAT = {0: 0} +STEP = 28800 + + +class CascadeRefineOperator(mx.operator.CustomOp): + def __init__(self, stride=0, network='', dataset='', prefix=''): + super(CascadeRefineOperator, self).__init__() + self.stride = int(stride) + self.prefix = prefix + generate_config(network, dataset) + self.mode = config.TRAIN.OHEM_MODE #0 for random 10:245, 1 for 10:246, 2 for 10:30, mode 1 for default + stride = self.stride + sstride = str(stride) + base_size = config.RPN_ANCHOR_CFG[sstride]['BASE_SIZE'] + allowed_border = config.RPN_ANCHOR_CFG[sstride]['ALLOWED_BORDER'] + ratios = config.RPN_ANCHOR_CFG[sstride]['RATIOS'] + scales = config.RPN_ANCHOR_CFG[sstride]['SCALES'] + base_anchors = generate_anchors(base_size=base_size, + ratios=list(ratios), + scales=np.array(scales, + dtype=np.float32), + stride=stride, + dense_anchor=config.DENSE_ANCHOR) + num_anchors = base_anchors.shape[0] + feat_height, feat_width = config.SCALES[0][ + 0] // self.stride, config.SCALES[0][0] // self.stride + feat_stride = self.stride + + A = num_anchors + K = feat_height * feat_width + self.A = A + + all_anchors = anchors_plane(feat_height, feat_width, feat_stride, + base_anchors) + all_anchors = all_anchors.reshape((K * A, 4)) + self.ori_anchors = all_anchors + self.nbatch = 0 + global STAT + for k in config.RPN_FEAT_STRIDE: + STAT[k] = [0, 0, 0] + + def apply_bbox_pred(self, bbox_pred, ind=None): + box_deltas = bbox_pred + box_deltas[:, 0::4] = box_deltas[:, 0::4] * config.TRAIN.BBOX_STDS[0] + box_deltas[:, 1::4] = box_deltas[:, 1::4] * config.TRAIN.BBOX_STDS[1] + box_deltas[:, 2::4] = box_deltas[:, 2::4] * config.TRAIN.BBOX_STDS[2] + box_deltas[:, 3::4] = box_deltas[:, 3::4] * config.TRAIN.BBOX_STDS[3] + if ind is None: + boxes = self.ori_anchors + else: + boxes = self.ori_anchors[ind] + #print('in apply',self.stride, box_deltas.shape, boxes.shape) + + widths = boxes[:, 2] - boxes[:, 0] + 1.0 + heights = boxes[:, 3] - boxes[:, 1] + 1.0 + ctr_x = boxes[:, 0] + 0.5 * (widths - 1.0) + ctr_y = boxes[:, 1] + 0.5 * (heights - 1.0) + + dx = box_deltas[:, 0:1] + dy = box_deltas[:, 1:2] + dw = box_deltas[:, 2:3] + dh = box_deltas[:, 3:4] + + pred_ctr_x = dx * widths[:, np.newaxis] + ctr_x[:, np.newaxis] + pred_ctr_y = dy * heights[:, np.newaxis] + ctr_y[:, np.newaxis] + pred_w = np.exp(dw) * widths[:, np.newaxis] + pred_h = np.exp(dh) * heights[:, np.newaxis] + + pred_boxes = np.zeros(box_deltas.shape) + # x1 + pred_boxes[:, 0:1] = pred_ctr_x - 0.5 * (pred_w - 1.0) + # y1 + pred_boxes[:, 1:2] = pred_ctr_y - 0.5 * (pred_h - 1.0) + # x2 + pred_boxes[:, 2:3] = pred_ctr_x + 0.5 * (pred_w - 1.0) + # y2 + pred_boxes[:, 3:4] = pred_ctr_y + 0.5 * (pred_h - 1.0) + return pred_boxes + + def assign_anchor_fpn(self, + gt_label, + anchors, + landmark=False, + prefix='face'): + IOU = config.TRAIN.CASCADE_OVERLAP + + gt_boxes = gt_label['gt_boxes'] + #_label = gt_label['gt_label'] + # clean up boxes + #nonneg = np.where(_label[:] != -1)[0] + #gt_boxes = gt_boxes[nonneg] + if landmark: + gt_landmarks = gt_label['gt_landmarks'] + #gt_landmarks = gt_landmarks[nonneg] + assert gt_boxes.shape[0] == gt_landmarks.shape[0] + #scales = np.array(scales, dtype=np.float32) + feat_strides = config.RPN_FEAT_STRIDE + bbox_pred_len = 4 + landmark_pred_len = 10 + num_anchors = anchors.shape[0] + A = self.A + total_anchors = num_anchors + feat_height, feat_width = config.SCALES[0][ + 0] // self.stride, config.SCALES[0][0] // self.stride + + #print('total_anchors', anchors.shape[0], len(inds_inside), file=sys.stderr) + + # label: 1 is positive, 0 is negative, -1 is dont care + labels = np.empty((num_anchors, ), dtype=np.float32) + labels.fill(-1) + #print('BB', anchors.shape, len(inds_inside)) + #print('gt_boxes', gt_boxes.shape, file=sys.stderr) + #tb = datetime.datetime.now() + #self._times[0] += (tb-ta).total_seconds() + #ta = datetime.datetime.now() + + if gt_boxes.size > 0: + # overlap between the anchors and the gt boxes + # overlaps (ex, gt) + overlaps = bbox_overlaps(anchors.astype(np.float), + gt_boxes.astype(np.float)) + argmax_overlaps = overlaps.argmax(axis=1) + #print('AAA', argmax_overlaps.shape) + max_overlaps = overlaps[np.arange(num_anchors), argmax_overlaps] + gt_argmax_overlaps = overlaps.argmax(axis=0) + gt_max_overlaps = overlaps[gt_argmax_overlaps, + np.arange(overlaps.shape[1])] + gt_argmax_overlaps = np.where(overlaps == gt_max_overlaps)[0] + + if not config.TRAIN.RPN_CLOBBER_POSITIVES: + # assign bg labels first so that positive labels can clobber them + labels[max_overlaps < IOU[0]] = 0 + + # fg label: for each gt, anchor with highest overlap + if config.TRAIN.RPN_FORCE_POSITIVE: + labels[gt_argmax_overlaps] = 1 + + # fg label: above threshold IoU + labels[max_overlaps >= IOU[1]] = 1 + + if config.TRAIN.RPN_CLOBBER_POSITIVES: + # assign bg labels last so that negative labels can clobber positives + labels[max_overlaps < IOU[0]] = 0 + else: + labels[:] = 0 + fg_inds = np.where(labels == 1)[0] + #print('fg count', len(fg_inds)) + + # subsample positive labels if we have too many + if config.TRAIN.RPN_ENABLE_OHEM == 0: + fg_inds = np.where(labels == 1)[0] + num_fg = int(config.TRAIN.RPN_FG_FRACTION * + config.TRAIN.RPN_BATCH_SIZE) + if len(fg_inds) > num_fg: + disable_inds = npr.choice(fg_inds, + size=(len(fg_inds) - num_fg), + replace=False) + if DEBUG: + disable_inds = fg_inds[:(len(fg_inds) - num_fg)] + labels[disable_inds] = -1 + + # subsample negative labels if we have too many + num_bg = config.TRAIN.RPN_BATCH_SIZE - np.sum(labels == 1) + bg_inds = np.where(labels == 0)[0] + if len(bg_inds) > num_bg: + disable_inds = npr.choice(bg_inds, + size=(len(bg_inds) - num_bg), + replace=False) + if DEBUG: + disable_inds = bg_inds[:(len(bg_inds) - num_bg)] + labels[disable_inds] = -1 + + #fg_inds = np.where(labels == 1)[0] + #num_fg = len(fg_inds) + #num_bg = num_fg*int(1.0/config.TRAIN.RPN_FG_FRACTION-1) + + #bg_inds = np.where(labels == 0)[0] + #if len(bg_inds) > num_bg: + # disable_inds = npr.choice(bg_inds, size=(len(bg_inds) - num_bg), replace=False) + # if DEBUG: + # disable_inds = bg_inds[:(len(bg_inds) - num_bg)] + # labels[disable_inds] = -1 + else: + fg_inds = np.where(labels == 1)[0] + num_fg = len(fg_inds) + bg_inds = np.where(labels == 0)[0] + num_bg = len(bg_inds) + + #print('anchor stat', num_fg, num_bg) + + bbox_targets = np.zeros((num_anchors, bbox_pred_len), dtype=np.float32) + if gt_boxes.size > 0: + #print('GT', gt_boxes.shape, gt_boxes[argmax_overlaps, :4].shape) + bbox_targets[:, :] = bbox_transform(anchors, + gt_boxes[argmax_overlaps, :]) + #bbox_targets[:,4] = gt_blur + #tb = datetime.datetime.now() + #self._times[1] += (tb-ta).total_seconds() + #ta = datetime.datetime.now() + + bbox_weights = np.zeros((num_anchors, bbox_pred_len), dtype=np.float32) + #bbox_weights[labels == 1, :] = np.array(config.TRAIN.RPN_BBOX_WEIGHTS) + bbox_weights[labels == 1, 0:4] = 1.0 + if bbox_pred_len > 4: + bbox_weights[labels == 1, 4:bbox_pred_len] = 0.1 + + if landmark: + landmark_targets = np.zeros((num_anchors, landmark_pred_len), + dtype=np.float32) + landmark_weights = np.zeros((num_anchors, landmark_pred_len), + dtype=np.float32) + #landmark_weights[labels == 1, :] = np.array(config.TRAIN.RPN_LANDMARK_WEIGHTS) + if landmark_pred_len == 10: + landmark_weights[labels == 1, :] = 1.0 + elif landmark_pred_len == 15: + v = [1.0, 1.0, 0.1] * 5 + assert len(v) == 15 + landmark_weights[labels == 1, :] = np.array(v) + else: + assert False + #TODO here + if gt_landmarks.size > 0: + #print('AAA',argmax_overlaps) + a_landmarks = gt_landmarks[argmax_overlaps, :, :] + landmark_targets[:] = landmark_transform(anchors, a_landmarks) + invalid = np.where(a_landmarks[:, 0, 2] < 0.0)[0] + #assert len(invalid)==0 + #landmark_weights[invalid, :] = np.array(config.TRAIN.RPN_INVALID_LANDMARK_WEIGHTS) + landmark_weights[invalid, :] = 0.0 + #tb = datetime.datetime.now() + #self._times[2] += (tb-ta).total_seconds() + #ta = datetime.datetime.now() + bbox_targets[:, + 0::4] = bbox_targets[:, 0::4] / config.TRAIN.BBOX_STDS[0] + bbox_targets[:, + 1::4] = bbox_targets[:, 1::4] / config.TRAIN.BBOX_STDS[1] + bbox_targets[:, + 2::4] = bbox_targets[:, 2::4] / config.TRAIN.BBOX_STDS[2] + bbox_targets[:, + 3::4] = bbox_targets[:, 3::4] / config.TRAIN.BBOX_STDS[3] + + #print('CC', anchors.shape, len(inds_inside)) + label = {} + _label = labels.reshape( + (1, feat_height, feat_width, A)).transpose(0, 3, 1, 2) + _label = _label.reshape((1, A * feat_height * feat_width)) + bbox_target = bbox_targets.reshape( + (1, feat_height * feat_width, + A * bbox_pred_len)).transpose(0, 2, 1) + bbox_weight = bbox_weights.reshape( + (1, feat_height * feat_width, A * bbox_pred_len)).transpose( + (0, 2, 1)) + label['%s_label' % prefix] = _label[0] + label['%s_bbox_target' % prefix] = bbox_target[0] + label['%s_bbox_weight' % prefix] = bbox_weight[0] + if landmark: + landmark_target = landmark_target.reshape( + (1, feat_height * feat_width, + A * landmark_pred_len)).transpose(0, 2, 1) + landmark_target /= config.TRAIN.LANDMARK_STD + landmark_weight = landmark_weight.reshape( + (1, feat_height * feat_width, + A * landmark_pred_len)).transpose((0, 2, 1)) + label['%s_landmark_target' % prefix] = landmark_target[0] + label['%s_landmark_weight' % prefix] = landmark_weight[0] + + return label + + def forward(self, is_train, req, in_data, out_data, aux): + self.nbatch += 1 + ta = datetime.datetime.now() + global STAT + A = config.NUM_ANCHORS + + cls_label_t0 = in_data[0].asnumpy() #BS, AHW + cls_score_t0 = in_data[1].asnumpy() #BS, C, AHW + cls_score = in_data[2].asnumpy() #BS, C, AHW + #labels_raw = in_data[1].asnumpy() #BS, ANCHORS + bbox_pred_t0 = in_data[3].asnumpy() #BS, AC, HW + bbox_target_t0 = in_data[4].asnumpy() #BS, AC, HW + cls_label_raw = in_data[5].asnumpy() #BS, AHW + gt_boxes = in_data[6].asnumpy() #BS, N, C=4+1 + #imgs = in_data[7].asnumpy().astype(np.uint8) + + batch_size = cls_score.shape[0] + num_anchors = cls_score.shape[2] + #print('in cas', cls_score.shape, bbox_target.shape) + + labels_out = np.zeros(shape=(batch_size, num_anchors), + dtype=np.float32) + bbox_target_out = np.zeros(shape=bbox_target_t0.shape, + dtype=np.float32) + anchor_weight = np.zeros((batch_size, num_anchors, 1), + dtype=np.float32) + valid_count = np.zeros((batch_size, 1), dtype=np.float32) + + bbox_pred_t0 = bbox_pred_t0.transpose((0, 2, 1)) + bbox_pred_t0 = bbox_pred_t0.reshape( + (bbox_pred_t0.shape[0], -1, 4)) #BS, H*W*A, C + bbox_target_t0 = bbox_target_t0.transpose((0, 2, 1)) + bbox_target_t0 = bbox_target_t0.reshape( + (bbox_target_t0.shape[0], -1, 4)) + + #print('anchor_weight', anchor_weight.shape) + + #assert labels.shape[0]==1 + #assert cls_score.shape[0]==1 + #assert bbox_weight.shape[0]==1 + #print('shape', cls_score.shape, labels.shape, file=sys.stderr) + #print('bbox_weight 0', bbox_weight.shape, file=sys.stderr) + #bbox_weight = np.zeros( (labels_raw.shape[0], labels_raw.shape[1], 4), dtype=np.float32) + _stat = [0, 0, 0] + SEL_TOPK = config.TRAIN.RPN_BATCH_SIZE + FAST = False + for ibatch in range(batch_size): + #bgr = imgs[ibatch].transpose( (1,2,0) )[:,:,::-1] + + if not FAST: + _gt_boxes = gt_boxes[ibatch] #N, 4+1 + _gtind = len(np.where(_gt_boxes[:, 4] >= 0)[0]) + #print('gt num', _gtind) + _gt_boxes = _gt_boxes[0:_gtind, :] + + #anchors_t1 = self.ori_anchors.copy() + #_cls_label_raw = cls_label_raw[ibatch] #AHW + #_cls_label_raw = _cls_label_raw.reshape( (A, -1) ).transpose( (1,0) ).reshape( (-1,) ) #HWA + #fg_ind_raw = np.where(_cls_label_raw>0)[0] + #_bbox_target_t0 = bbox_target_t0[ibatch][fg_ind_raw] + #_bbox_pred_t0 = bbox_pred_t0[ibatch][fg_ind_raw] + #anchors_t1_pos = self.apply_bbox_pred(_bbox_pred_t0, ind=fg_ind_raw) + #anchors_t1[fg_ind_raw,:] = anchors_t1_pos + + anchors_t1 = self.apply_bbox_pred(bbox_pred_t0[ibatch]) + assert anchors_t1.shape[0] == self.ori_anchors.shape[0] + + #for i in range(_gt_boxes.shape[0]): + # box = _gt_boxes[i].astype(np.int) + # print('%d: gt%d'%(self.nbatch, i), box) + # #color = (0,0,255) + # #cv2.rectangle(img, (box[0], box[1]), (box[2], box[3]), color, 2) + #for i in range(anchors_t1.shape[0]): + # box1 = self.ori_anchors[i].astype(np.int) + # box2 = anchors_t1[i].astype(np.int) + # print('%d %d: anchorscompare %d'%(self.nbatch, self.stride, i), box1, box2) + #color = (255,255,0) + #cv2.rectangle(img, (box[0], box[1]), (box[2], box[3]), color, 2) + #filename = "./debug/%d_%d_%d.jpg"%(self.nbatch, ibatch, stride) + #cv2.imwrite(filename, img) + #print(filename) + #gt_label = {'gt_boxes': gt_anchors, 'gt_label' : labels_raw[ibatch]} + gt_label = {'gt_boxes': _gt_boxes} + new_label_dict = self.assign_anchor_fpn(gt_label, + anchors_t1, + False, + prefix=self.prefix) + labels = new_label_dict['%s_label' % self.prefix] #AHW + new_bbox_target = new_label_dict['%s_bbox_target' % + self.prefix] #AC,HW + #print('assign ret', labels.shape, new_bbox_target.shape) + _anchor_weight = np.zeros((num_anchors, 1), dtype=np.float32) + fg_score = cls_score[ibatch, 1, :] - cls_score[ibatch, 0, :] + fg_inds = np.where(labels > 0)[0] + num_fg = int(config.TRAIN.RPN_FG_FRACTION * + config.TRAIN.RPN_BATCH_SIZE) + origin_num_fg = len(fg_inds) + #continue + #print('cas fg', len(fg_inds), num_fg, file=sys.stderr) + if len(fg_inds) > num_fg: + if self.mode == 0: + disable_inds = np.random.choice(fg_inds, + size=(len(fg_inds) - + num_fg), + replace=False) + labels[disable_inds] = -1 + else: + pos_ohem_scores = fg_score[fg_inds] + order_pos_ohem_scores = pos_ohem_scores.ravel( + ).argsort() + sampled_inds = fg_inds[order_pos_ohem_scores[:num_fg]] + labels[fg_inds] = -1 + labels[sampled_inds] = 1 + + n_fg = np.sum(labels > 0) + fg_inds = np.where(labels > 0)[0] + num_bg = config.TRAIN.RPN_BATCH_SIZE - n_fg + if self.mode == 2: + num_bg = max( + 48, n_fg * int(1.0 / config.TRAIN.RPN_FG_FRACTION - 1)) + + bg_inds = np.where(labels == 0)[0] + origin_num_bg = len(bg_inds) + if num_bg == 0: + labels[bg_inds] = -1 + elif len(bg_inds) > num_bg: + # sort ohem scores + + if self.mode == 0: + disable_inds = np.random.choice(bg_inds, + size=(len(bg_inds) - + num_bg), + replace=False) + labels[disable_inds] = -1 + else: + neg_ohem_scores = fg_score[bg_inds] + order_neg_ohem_scores = neg_ohem_scores.ravel( + ).argsort()[::-1] + sampled_inds = bg_inds[order_neg_ohem_scores[:num_bg]] + #print('sampled_inds_bg', sampled_inds, file=sys.stderr) + labels[bg_inds] = -1 + labels[sampled_inds] = 0 + + if n_fg > 0: + order0_labels = labels.reshape((1, A, -1)).transpose( + (0, 2, 1)).reshape((-1, )) + bbox_fg_inds = np.where(order0_labels > 0)[0] + #print('bbox_fg_inds, order0 ', bbox_fg_inds, file=sys.stderr) + _anchor_weight[bbox_fg_inds, :] = 1.0 + anchor_weight[ibatch] = _anchor_weight + valid_count[ibatch][0] = n_fg + labels_out[ibatch] = labels + #print('labels_out', self.stride, ibatch, labels) + bbox_target_out[ibatch] = new_bbox_target + #print('cascade stat', self.stride, ibatch, len(labels), len(np.where(labels==1)[0]), len(np.where(labels==0)[0])) + else: #FAST MODE + fg_score_t0 = cls_score_t0[ibatch, 1, :] - cls_score_t0[ibatch, + 0, :] + sort_idx_t0 = np.argsort( + fg_score_t0.flatten())[::-1][0:SEL_TOPK] + _bbox_pred_t0 = bbox_pred_t0[ibatch][sort_idx_t0] + _bbox_target_t0 = bbox_target_t0[ibatch][sort_idx_t0] + #print('SEL fg score:', fg_score_t0[sort_idx[-1]], fg_score_t0[sort_idx[0]]) + anchors_t0 = self.apply_bbox_pred(_bbox_pred_t0) + gt_anchors = self.apply_bbox_pred(_bbox_target_t0) + #gt_label = {'gt_boxes': gt_anchors, 'gt_label' : labels_raw[ibatch]} + gt_label = {'gt_boxes': gt_anchors} + new_label_dict = self.assign_anchor_fpn(gt_label, + anchors_t0, + False, + prefix=self.prefix) + labels = new_label_dict['%s_label' % self.prefix] + new_bbox_target = new_label_dict['%s_bbox_target' % + self.prefix] + #print('assign ret', labels.shape, new_bbox_target.shape) + _anchor_weight = np.zeros((num_anchors, 1), dtype=np.float32) + fg_score = cls_score[ibatch, 1, :] - cls_score[ibatch, 0, :] + fg_inds = np.where(labels > 0)[0] + _labels = np.empty(shape=labels.shape, dtype=np.float32) + _labels.fill(-1) + _labels[sort_idx_idx] = labels + + anchor_weight[ibatch] = _anchor_weight + valid_count[ibatch][0] = len(fg_inds) + labels_out[ibatch] = _labels + #print('labels_out', self.stride, ibatch, labels) + bbox_target_out[ibatch] = new_bbox_target + + #print('cascade pos stat', self.stride, batch_size, np.sum(valid_count)) + for ind, val in enumerate( + [labels_out, bbox_target_out, anchor_weight, valid_count]): + val = mx.nd.array(val) + self.assign(out_data[ind], req[ind], val) + tb = datetime.datetime.now() + #print('cascade forward cost', self.stride, (tb-ta).total_seconds()) + + def backward(self, req, out_grad, in_data, out_data, in_grad, aux): + for i in range(len(in_grad)): + self.assign(in_grad[i], req[i], 0) + + +@mx.operator.register('cascade_refine') +class CascadeRefineProp(mx.operator.CustomOpProp): + def __init__(self, stride=0, network='', dataset='', prefix=''): + super(CascadeRefineProp, self).__init__(need_top_grad=False) + self.stride = stride + self.network = network + self.dataset = dataset + self.prefix = prefix + + def list_arguments(self): + #return ['cls_label_t0', 'cls_pred_t0', 'cls_pred', 'bbox_pred_t0', 'bbox_label_t0', 'cls_label_raw', 'cas_gt_boxes', 'cas_img'] + return [ + 'cls_label_t0', 'cls_pred_t0', 'cls_pred', 'bbox_pred_t0', + 'bbox_label_t0', 'cls_label_raw', 'cas_gt_boxes' + ] + + def list_outputs(self): + return [ + 'cls_label_out', 'bbox_label_out', 'anchor_weight_out', + 'pos_count_out' + ] + + def infer_shape(self, in_shape): + cls_pred_shape = in_shape[1] + bs = cls_pred_shape[0] + num_anchors = cls_pred_shape[2] + #print('in_rpn_ohem', in_shape[0], in_shape[1], in_shape[2], file=sys.stderr) + #print('in_rpn_ohem', labels_shape, anchor_weight_shape) + cls_label_shape = [bs, num_anchors] + + return in_shape, \ + [cls_label_shape, in_shape[4], [bs,num_anchors,1], [bs,1]] + + def create_operator(self, ctx, shapes, dtypes): + return CascadeRefineOperator(self.stride, self.network, self.dataset, + self.prefix) + + def declare_backward_dependency(self, out_grad, in_data, out_data): + return [] diff --git a/detection/RetinaFace/rcnn/PY_OP/rpn_fpn_ohem3.py b/detection/RetinaFace/rcnn/PY_OP/rpn_fpn_ohem3.py new file mode 100644 index 0000000..b8f7d46 --- /dev/null +++ b/detection/RetinaFace/rcnn/PY_OP/rpn_fpn_ohem3.py @@ -0,0 +1,175 @@ +from __future__ import print_function +import sys +import mxnet as mx +import numpy as np +from distutils.util import strtobool +from ..config import config, generate_config + +STAT = {0: 0} +STEP = 28800 + + +class RPNFPNOHEM3Operator(mx.operator.CustomOp): + def __init__(self, stride=0, network='', dataset='', prefix=''): + super(RPNFPNOHEM3Operator, self).__init__() + self.stride = int(stride) + self.prefix = prefix + generate_config(network, dataset) + self.mode = config.TRAIN.OHEM_MODE #0 for random 10:245, 1 for 10:246, 2 for 10:30, mode 1 for default + global STAT + for k in config.RPN_FEAT_STRIDE: + STAT[k] = [0, 0, 0] + + def forward(self, is_train, req, in_data, out_data, aux): + global STAT + + cls_score = in_data[0].asnumpy() #BS, 2, ANCHORS + labels_raw = in_data[1].asnumpy() # BS, ANCHORS + + A = config.NUM_ANCHORS + anchor_weight = np.zeros((labels_raw.shape[0], labels_raw.shape[1], 1), + dtype=np.float32) + valid_count = np.zeros((labels_raw.shape[0], 1), dtype=np.float32) + #print('anchor_weight', anchor_weight.shape) + + #assert labels.shape[0]==1 + #assert cls_score.shape[0]==1 + #assert bbox_weight.shape[0]==1 + #print('shape', cls_score.shape, labels.shape, file=sys.stderr) + #print('bbox_weight 0', bbox_weight.shape, file=sys.stderr) + #bbox_weight = np.zeros( (labels_raw.shape[0], labels_raw.shape[1], 4), dtype=np.float32) + _stat = [0, 0, 0] + for ibatch in range(labels_raw.shape[0]): + _anchor_weight = np.zeros((labels_raw.shape[1], 1), + dtype=np.float32) + labels = labels_raw[ibatch] + fg_score = cls_score[ibatch, 1, :] - cls_score[ibatch, 0, :] + + fg_inds = np.where(labels > 0)[0] + num_fg = int(config.TRAIN.RPN_FG_FRACTION * + config.TRAIN.RPN_BATCH_SIZE) + origin_num_fg = len(fg_inds) + #print(len(fg_inds), num_fg, file=sys.stderr) + if len(fg_inds) > num_fg: + if self.mode == 0: + disable_inds = np.random.choice(fg_inds, + size=(len(fg_inds) - + num_fg), + replace=False) + labels[disable_inds] = -1 + else: + pos_ohem_scores = fg_score[fg_inds] + order_pos_ohem_scores = pos_ohem_scores.ravel().argsort() + sampled_inds = fg_inds[order_pos_ohem_scores[:num_fg]] + labels[fg_inds] = -1 + labels[sampled_inds] = 1 + + n_fg = np.sum(labels > 0) + fg_inds = np.where(labels > 0)[0] + num_bg = config.TRAIN.RPN_BATCH_SIZE - n_fg + if self.mode == 2: + num_bg = max( + 48, n_fg * int(1.0 / config.TRAIN.RPN_FG_FRACTION - 1)) + + bg_inds = np.where(labels == 0)[0] + origin_num_bg = len(bg_inds) + if num_bg == 0: + labels[bg_inds] = -1 + elif len(bg_inds) > num_bg: + # sort ohem scores + + if self.mode == 0: + disable_inds = np.random.choice(bg_inds, + size=(len(bg_inds) - + num_bg), + replace=False) + labels[disable_inds] = -1 + else: + neg_ohem_scores = fg_score[bg_inds] + order_neg_ohem_scores = neg_ohem_scores.ravel().argsort( + )[::-1] + sampled_inds = bg_inds[order_neg_ohem_scores[:num_bg]] + #print('sampled_inds_bg', sampled_inds, file=sys.stderr) + labels[bg_inds] = -1 + labels[sampled_inds] = 0 + + if n_fg > 0: + order0_labels = labels.reshape((1, A, -1)).transpose( + (0, 2, 1)).reshape((-1, )) + bbox_fg_inds = np.where(order0_labels > 0)[0] + #print('bbox_fg_inds, order0 ', bbox_fg_inds, file=sys.stderr) + _anchor_weight[bbox_fg_inds, :] = 1.0 + anchor_weight[ibatch] = _anchor_weight + valid_count[ibatch][0] = n_fg + + #if self.prefix=='face': + # #print('fg-bg', self.stride, n_fg, num_bg) + # STAT[0]+=1 + # STAT[self.stride][0] += config.TRAIN.RPN_BATCH_SIZE + # STAT[self.stride][1] += n_fg + # STAT[self.stride][2] += np.sum(fg_score[fg_inds]>=0) + # #_stat[0] += config.TRAIN.RPN_BATCH_SIZE + # #_stat[1] += n_fg + # #_stat[2] += np.sum(fg_score[fg_inds]>=0) + # #print('stride num_fg', self.stride, n_fg, file=sys.stderr) + # #ACC[self.stride] += np.sum(fg_score[fg_inds]>=0) + # #x = float(labels_raw.shape[0]*len(config.RPN_FEAT_STRIDE)) + # x = 1.0 + # if STAT[0]%STEP==0: + # _str = ['STAT'] + # STAT[0] = 0 + # for k in config.RPN_FEAT_STRIDE: + # acc = float(STAT[k][2])/STAT[k][1] + # acc0 = float(STAT[k][1])/STAT[k][0] + # #_str.append("%d: all-fg(%d, %d, %.4f), fg-fgcorrect(%d, %d, %.4f)"%(k,STAT[k][0], STAT[k][1], acc0, STAT[k][1], STAT[k][2], acc)) + # _str.append("%d: (%d, %d, %.4f)"%(k, STAT[k][1], STAT[k][2], acc)) + # STAT[k] = [0,0,0] + # _str = ' | '.join(_str) + # print(_str, file=sys.stderr) + #if self.stride==4 and num_fg>0: + # print('_stat_', self.stride, num_fg, num_bg, file=sys.stderr) + + #labels_ohem = mx.nd.array(labels_raw) + #anchor_weight = mx.nd.array(anchor_weight) + #print('valid_count', self.stride, np.sum(valid_count)) + #print('_stat', _stat, valid_count) + + for ind, val in enumerate([labels_raw, anchor_weight, valid_count]): + val = mx.nd.array(val) + self.assign(out_data[ind], req[ind], val) + + def backward(self, req, out_grad, in_data, out_data, in_grad, aux): + for i in range(len(in_grad)): + self.assign(in_grad[i], req[i], 0) + + +@mx.operator.register('rpn_fpn_ohem3') +class RPNFPNOHEM3Prop(mx.operator.CustomOpProp): + def __init__(self, stride=0, network='', dataset='', prefix=''): + super(RPNFPNOHEM3Prop, self).__init__(need_top_grad=False) + self.stride = stride + self.network = network + self.dataset = dataset + self.prefix = prefix + + def list_arguments(self): + return ['cls_score', 'labels'] + + def list_outputs(self): + return ['labels_ohem', 'anchor_weight', 'valid_count'] + + def infer_shape(self, in_shape): + labels_shape = in_shape[1] + #print('in_rpn_ohem', in_shape[0], in_shape[1], in_shape[2], file=sys.stderr) + anchor_weight_shape = [labels_shape[0], labels_shape[1], 1] + #print('in_rpn_ohem', labels_shape, anchor_weight_shape) + + return in_shape, \ + [labels_shape, anchor_weight_shape, [labels_shape[0], 1]] + + def create_operator(self, ctx, shapes, dtypes): + return RPNFPNOHEM3Operator(self.stride, self.network, self.dataset, + self.prefix) + + def declare_backward_dependency(self, out_grad, in_data, out_data): + return [] diff --git a/RetinaFace/rcnn/__init__.py b/detection/RetinaFace/rcnn/__init__.py similarity index 100% rename from RetinaFace/rcnn/__init__.py rename to detection/RetinaFace/rcnn/__init__.py diff --git a/RetinaFace/rcnn/core/__init__.py b/detection/RetinaFace/rcnn/core/__init__.py similarity index 100% rename from RetinaFace/rcnn/core/__init__.py rename to detection/RetinaFace/rcnn/core/__init__.py diff --git a/detection/RetinaFace/rcnn/core/callback.py b/detection/RetinaFace/rcnn/core/callback.py new file mode 100644 index 0000000..729a392 --- /dev/null +++ b/detection/RetinaFace/rcnn/core/callback.py @@ -0,0 +1,16 @@ +import mxnet as mx + + +def do_checkpoint(prefix, means, stds): + def _callback(iter_no, sym, arg, aux): + if 'bbox_pred_weight' in arg: + arg['bbox_pred_weight_test'] = (arg['bbox_pred_weight'].T * + mx.nd.array(stds)).T + arg['bbox_pred_bias_test'] = arg['bbox_pred_bias'] * mx.nd.array( + stds) + mx.nd.array(means) + mx.model.save_checkpoint(prefix, iter_no + 1, sym, arg, aux) + if 'bbox_pred_weight' in arg: + arg.pop('bbox_pred_weight_test') + arg.pop('bbox_pred_bias_test') + + return _callback diff --git a/RetinaFace/rcnn/core/loader.py b/detection/RetinaFace/rcnn/core/loader.py similarity index 61% rename from RetinaFace/rcnn/core/loader.py rename to detection/RetinaFace/rcnn/core/loader.py index 019ae75..1d34d2e 100644 --- a/RetinaFace/rcnn/core/loader.py +++ b/detection/RetinaFace/rcnn/core/loader.py @@ -14,7 +14,13 @@ from rcnn.io.rpn import get_rpn_testbatch, get_rpn_batch, assign_anchor_fpn, get class CropLoader(mx.io.DataIter): - def __init__(self, feat_sym, roidb, batch_size=1, shuffle=False, ctx=None, work_load_list=None, + def __init__(self, + feat_sym, + roidb, + batch_size=1, + shuffle=False, + ctx=None, + work_load_list=None, aspect_grouping=False): """ This Iter will provide roi data to Fast R-CNN network @@ -61,19 +67,24 @@ class CropLoader(mx.io.DataIter): self.label_name = [] prefixes = ['face'] if config.HEAD_BOX: - prefixes.append('head') + prefixes.append('head') names = [] for prefix in prefixes: - names += [prefix+'_label', prefix+'_bbox_target', prefix+'_bbox_weight'] - if prefix=='face' and config.FACE_LANDMARK: - names += [prefix+'_landmark_target', prefix+'_landmark_weight'] + names += [ + prefix + '_label', prefix + '_bbox_target', + prefix + '_bbox_weight' + ] + if prefix == 'face' and config.FACE_LANDMARK: + names += [ + prefix + '_landmark_target', prefix + '_landmark_weight' + ] #names = ['label', 'bbox_weight'] for stride in self.feat_stride: - for n in names: - k = "%s_stride%d"%(n,stride) - self.label_name.append(k) - if config.CASCADE>0: - self.label_name.append('gt_boxes') + for n in names: + k = "%s_stride%d" % (n, stride) + self.label_name.append(k) + if config.CASCADE > 0: + self.label_name.append('gt_boxes') # status variable for synchronization between get_data and get_label self.cur = 0 @@ -82,7 +93,8 @@ class CropLoader(mx.io.DataIter): self.label = None # infer shape feat_shape_list = [] - _data_shape = [('data', (1, 3, max([v[1] for v in config.SCALES]), max([v[1] for v in config.SCALES])))] + _data_shape = [('data', (1, 3, max([v[1] for v in config.SCALES]), + max([v[1] for v in config.SCALES])))] _data_shape = dict(_data_shape) for i in range(len(self.feat_stride)): _, feat_shape, _ = self.feat_sym[i].infer_shape(**_data_shape) @@ -109,7 +121,7 @@ class CropLoader(mx.io.DataIter): def reset(self): self.cur = 0 if self.shuffle: - np.random.shuffle(self.index) + np.random.shuffle(self.index) def iter_next(self): return self.cur + self.batch_size <= self.size @@ -118,9 +130,12 @@ class CropLoader(mx.io.DataIter): if self.iter_next(): self.get_batch() self.cur += self.batch_size - return mx.io.DataBatch(data=self.data, label=self.label, - pad=self.getpad(), index=self.getindex(), - provide_data=self.provide_data, provide_label=self.provide_label) + return mx.io.DataBatch(data=self.data, + label=self.label, + pad=self.getpad(), + index=self.getindex(), + provide_data=self.provide_data, + provide_label=self.provide_label) else: raise StopIteration @@ -142,36 +157,43 @@ class CropLoader(mx.io.DataIter): max_shapes = dict(max_data_shape + max_label_shape) input_batch_size = max_shapes['data'][0] dummy_boxes = np.zeros((0, 5)) - dummy_info = [ [max_shapes['data'][2], max_shapes['data'][3], 1.0] ] - dummy_label = {'gt_boxes' : dummy_boxes} - dummy_blur = np.zeros((0,)) + dummy_info = [[max_shapes['data'][2], max_shapes['data'][3], 1.0]] + dummy_label = {'gt_boxes': dummy_boxes} + dummy_blur = np.zeros((0, )) dummy_label['gt_blur'] = dummy_blur - label_dict = {} if config.HEAD_BOX: - head_label_dict = self.aa.assign_anchor_fpn(dummy_label, dummy_info, False, prefix='head') - label_dict.update(head_label_dict) + head_label_dict = self.aa.assign_anchor_fpn(dummy_label, + dummy_info, + False, + prefix='head') + label_dict.update(head_label_dict) if config.FACE_LANDMARK: - dummy_landmarks = np.zeros( (0,5,3) ) - dummy_label['gt_landmarks'] = dummy_landmarks - face_label_dict = self.aa.assign_anchor_fpn(dummy_label, dummy_info, config.FACE_LANDMARK, prefix='face') + dummy_landmarks = np.zeros((0, 5, 3)) + dummy_label['gt_landmarks'] = dummy_landmarks + face_label_dict = self.aa.assign_anchor_fpn(dummy_label, + dummy_info, + config.FACE_LANDMARK, + prefix='face') label_dict.update(face_label_dict) - if config.CASCADE>0: - label_dict['gt_boxes'] = np.zeros((0, config.TRAIN.MAX_BBOX_PER_IMAGE, 5), dtype=np.float32) + if config.CASCADE > 0: + label_dict['gt_boxes'] = np.zeros( + (0, config.TRAIN.MAX_BBOX_PER_IMAGE, 5), dtype=np.float32) label_list = [] for k in self.label_name: - label_list.append(label_dict[k]) - label_shape = [(k, tuple([input_batch_size] + list(v.shape[1:]))) for k, v in zip(self.label_name, label_list)] + label_list.append(label_dict[k]) + label_shape = [(k, tuple([input_batch_size] + list(v.shape[1:]))) + for k, v in zip(self.label_name, label_list)] return max_data_shape, label_shape def get_batch(self): # slice roidb cur_from = self.cur cur_to = min(cur_from + self.batch_size, self.size) - assert cur_to==cur_from+self.batch_size + assert cur_to == cur_from + self.batch_size roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)] # decide multi device slice @@ -203,7 +225,7 @@ class CropLoader(mx.io.DataIter): #iiddxx = 0 select_stride = 0 if config.RANDOM_FEAT_STRIDE: - select_stride = random.choice(config.RPN_FEAT_STRIDE) + select_stride = random.choice(config.RPN_FEAT_STRIDE) for data, label in zip(data_list, label_list): data_shape = {k: v.shape for k, v in data.items()} @@ -215,46 +237,60 @@ class CropLoader(mx.io.DataIter): feat_shape_list.append(feat_shape) im_info = data['im_info'] gt_boxes = label['gt_boxes'] - gt_label = {'gt_boxes':gt_boxes} + gt_label = {'gt_boxes': gt_boxes} if config.USE_BLUR: - gt_blur = label['gt_blur'] - gt_label['gt_blur'] = gt_blur + gt_blur = label['gt_blur'] + gt_label['gt_blur'] = gt_blur if self._debug: - img = data['data'].copy()[0].transpose( (1,2,0) )[:,:,::-1].copy() - print('DEBUG SHAPE', data['data'].shape, label['gt_boxes'].shape) + img = data['data'].copy()[0].transpose( + (1, 2, 0))[:, :, ::-1].copy() + print('DEBUG SHAPE', data['data'].shape, + label['gt_boxes'].shape) - box = label['gt_boxes'].copy()[0][0:4].astype(np.int) - cv2.rectangle(img, (box[0], box[1]), (box[2], box[3]), (0, 255, 0), 2) - filename = './debugout/%d.png' % (self._debug_id) - print('debug write', filename) - cv2.imwrite(filename, img) - self._debug_id+=1 - #print('DEBUG', img.shape, bbox.shape) + box = label['gt_boxes'].copy()[0][0:4].astype(np.int) + cv2.rectangle(img, (box[0], box[1]), (box[2], box[3]), + (0, 255, 0), 2) + filename = './debugout/%d.png' % (self._debug_id) + print('debug write', filename) + cv2.imwrite(filename, img) + self._debug_id += 1 + #print('DEBUG', img.shape, bbox.shape) label_dict = {} if config.HEAD_BOX: - head_label_dict = self.aa.assign_anchor_fpn(gt_label, im_info, False, prefix='head', select_stride = select_stride) - label_dict.update(head_label_dict) + head_label_dict = self.aa.assign_anchor_fpn( + gt_label, + im_info, + False, + prefix='head', + select_stride=select_stride) + label_dict.update(head_label_dict) if config.FACE_LANDMARK: - gt_landmarks = label['gt_landmarks'] - gt_label['gt_landmarks'] = gt_landmarks + gt_landmarks = label['gt_landmarks'] + gt_label['gt_landmarks'] = gt_landmarks #ta = datetime.datetime.now() #face_label_dict = assign_anchor_fpn(feat_shape_list, gt_label, im_info, config.FACE_LANDMARK, prefix='face', select_stride = select_stride) - face_label_dict = self.aa.assign_anchor_fpn(gt_label, im_info, config.FACE_LANDMARK, prefix='face', select_stride = select_stride) + face_label_dict = self.aa.assign_anchor_fpn( + gt_label, + im_info, + config.FACE_LANDMARK, + prefix='face', + select_stride=select_stride) #tb = datetime.datetime.now() #self._times[0] += (tb-ta).total_seconds() label_dict.update(face_label_dict) #for k in label_dict: # print(k, label_dict[k].shape) - if config.CASCADE>0: - pad_gt_boxes = np.empty( (1, config.TRAIN.MAX_BBOX_PER_IMAGE, 5), dtype=np.float32) - pad_gt_boxes.fill(-1) - pad_gt_boxes[0, 0:gt_boxes.shape[0],:] = gt_boxes - label_dict['gt_boxes'] = pad_gt_boxes + if config.CASCADE > 0: + pad_gt_boxes = np.empty( + (1, config.TRAIN.MAX_BBOX_PER_IMAGE, 5), dtype=np.float32) + pad_gt_boxes.fill(-1) + pad_gt_boxes[0, 0:gt_boxes.shape[0], :] = gt_boxes + label_dict['gt_boxes'] = pad_gt_boxes #print('im_info', im_info.shape) #print(gt_boxes.shape) for k in self.label_name: - label[k] = label_dict[k] + label[k] = label_dict[k] all_data = dict() for key in self.data_name: @@ -264,7 +300,8 @@ class CropLoader(mx.io.DataIter): for key in self.label_name: pad = 0 if key.startswith('bbox_') else -1 #print('label vstack', key, pad, len(label_list), file=sys.stderr) - all_label[key] = tensor_vstack([batch[key] for batch in label_list], pad=pad) + all_label[key] = tensor_vstack( + [batch[key] for batch in label_list], pad=pad) self.data = [mx.nd.array(all_data[key]) for key in self.data_name] self.label = [mx.nd.array(all_label[key]) for key in self.label_name] @@ -272,8 +309,15 @@ class CropLoader(mx.io.DataIter): # print('LABEL SHAPE', _label.shape) #print(self._times) + class CropLoader2(mx.io.DataIter): - def __init__(self, feat_sym, roidb, batch_size=1, shuffle=False, ctx=None, work_load_list=None, + def __init__(self, + feat_sym, + roidb, + batch_size=1, + shuffle=False, + ctx=None, + work_load_list=None, aspect_grouping=False): """ This Iter will provide roi data to Fast R-CNN network @@ -319,17 +363,22 @@ class CropLoader2(mx.io.DataIter): self.label_name = [] prefixes = ['face'] if config.HEAD_BOX: - prefixes.append('head') + prefixes.append('head') names = [] for prefix in prefixes: - names += [prefix+'_label', prefix+'_bbox_target', prefix+'_bbox_weight'] - if prefix=='face' and config.FACE_LANDMARK: - names += [prefix+'_landmark_target', prefix+'_landmark_weight'] + names += [ + prefix + '_label', prefix + '_bbox_target', + prefix + '_bbox_weight' + ] + if prefix == 'face' and config.FACE_LANDMARK: + names += [ + prefix + '_landmark_target', prefix + '_landmark_weight' + ] #names = ['label', 'bbox_weight'] for stride in self.feat_stride: - for n in names: - k = "%s_stride%d"%(n,stride) - self.label_name.append(k) + for n in names: + k = "%s_stride%d" % (n, stride) + self.label_name.append(k) # status variable for synchronization between get_data and get_label self.cur = 0 self.batch = None @@ -338,7 +387,9 @@ class CropLoader2(mx.io.DataIter): # get first batch to fill in provide_data and provide_label self.reset() - self.q_in = [multiprocessing.Queue(1024) for i in range(config.NUM_CPU)] + self.q_in = [ + multiprocessing.Queue(1024) for i in range(config.NUM_CPU) + ] #self.q_in = multiprocessing.Queue(1024) self.q_out = multiprocessing.Queue(1024) self.start() @@ -353,49 +404,52 @@ class CropLoader2(mx.io.DataIter): return [(k, v.shape) for k, v in zip(self.label_name, self.label)] def reset(self): - pass + pass @staticmethod def input_worker(q_in, roidb, batch_size): - index = np.arange(len(roidb)) - np.random.shuffle(index) - cur_from = 0 - while True: - cur_to = cur_from + batch_size - if cur_to>len(roidb): - np.random.shuffle(index) - cur_from = 0 - continue - _roidb = [roidb[index[i]] for i in range(cur_from, cur_to)] - istart = index[cur_from] - q_in[istart%len(q_in)].put(_roidb) - cur_from = cur_to + index = np.arange(len(roidb)) + np.random.shuffle(index) + cur_from = 0 + while True: + cur_to = cur_from + batch_size + if cur_to > len(roidb): + np.random.shuffle(index) + cur_from = 0 + continue + _roidb = [roidb[index[i]] for i in range(cur_from, cur_to)] + istart = index[cur_from] + q_in[istart % len(q_in)].put(_roidb) + cur_from = cur_to @staticmethod def gen_worker(q_in, q_out): - while True: - deq = q_in.get() - if deq is None: - break - _roidb = deq - data, label = get_crop_batch(_roidb) - print('generated') - q_out.put( (data, label) ) + while True: + deq = q_in.get() + if deq is None: + break + _roidb = deq + data, label = get_crop_batch(_roidb) + print('generated') + q_out.put((data, label)) def start(self): - input_process = multiprocessing.Process(target=CropLoader2.input_worker, args=(self.q_in, self.roidb, self.batch_size)) - #gen_process = multiprocessing.Process(target=gen_worker, args=(q_in, q_out)) - gen_process = [multiprocessing.Process(target=CropLoader2.gen_worker, args=(self.q_in[i], self.q_out)) \ - for i in range(config.NUM_CPU)] - input_process.start() - for p in gen_process: - p.start() - + input_process = multiprocessing.Process( + target=CropLoader2.input_worker, + args=(self.q_in, self.roidb, self.batch_size)) + #gen_process = multiprocessing.Process(target=gen_worker, args=(q_in, q_out)) + gen_process = [multiprocessing.Process(target=CropLoader2.gen_worker, args=(self.q_in[i], self.q_out)) \ + for i in range(config.NUM_CPU)] + input_process.start() + for p in gen_process: + p.start() def next(self): self.get_batch() - return mx.io.DataBatch(data=self.data, label=self.label, - provide_data=self.provide_data, provide_label=self.provide_label) + return mx.io.DataBatch(data=self.data, + label=self.label, + provide_data=self.provide_data, + provide_label=self.provide_label) def infer_shape(self, max_data_shape=None, max_label_shape=None): """ Return maximum data and label shape for single gpu """ @@ -406,8 +460,8 @@ class CropLoader2(mx.io.DataIter): max_shapes = dict(max_data_shape + max_label_shape) input_batch_size = max_shapes['data'][0] dummy_boxes = np.zeros((0, 5)) - dummy_info = [ [max_shapes['data'][2], max_shapes['data'][3], 1.0] ] - dummy_label = {'gt_boxes' : dummy_boxes} + dummy_info = [[max_shapes['data'][2], max_shapes['data'][3], 1.0]] + dummy_label = {'gt_boxes': dummy_boxes} # infer shape feat_shape_list = [] @@ -418,19 +472,28 @@ class CropLoader2(mx.io.DataIter): label_dict = {} if config.HEAD_BOX: - head_label_dict = assign_anchor_fpn(feat_shape_list, dummy_label, dummy_info, False, prefix='head') - label_dict.update(head_label_dict) + head_label_dict = assign_anchor_fpn(feat_shape_list, + dummy_label, + dummy_info, + False, + prefix='head') + label_dict.update(head_label_dict) if config.FACE_LANDMARK: - dummy_landmarks = np.zeros( (0,11) ) - dummy_label['gt_landmarks'] = dummy_landmarks - face_label_dict = assign_anchor_fpn(feat_shape_list, dummy_label, dummy_info, config.FACE_LANDMARK, prefix='face') + dummy_landmarks = np.zeros((0, 11)) + dummy_label['gt_landmarks'] = dummy_landmarks + face_label_dict = assign_anchor_fpn(feat_shape_list, + dummy_label, + dummy_info, + config.FACE_LANDMARK, + prefix='face') label_dict.update(face_label_dict) label_list = [] for k in self.label_name: - label_list.append(label_dict[k]) - label_shape = [(k, tuple([input_batch_size] + list(v.shape[1:]))) for k, v in zip(self.label_name, label_list)] + label_list.append(label_dict[k]) + label_shape = [(k, tuple([input_batch_size] + list(v.shape[1:]))) + for k, v in zip(self.label_name, label_list)] return max_data_shape, label_shape def get_batch(self): @@ -450,19 +513,27 @@ class CropLoader2(mx.io.DataIter): # label[k] = [0 for i in range(config.TRAIN.BATCH_IMAGES)] im_info = data['im_info'] gt_boxes = label['gt_boxes'] - gt_label = {'gt_boxes':gt_boxes} + gt_label = {'gt_boxes': gt_boxes} label_dict = {} - head_label_dict = assign_anchor_fpn(feat_shape_list, gt_label, im_info, False, prefix='head') + head_label_dict = assign_anchor_fpn(feat_shape_list, + gt_label, + im_info, + False, + prefix='head') label_dict.update(head_label_dict) if config.FACE_LANDMARK: - gt_landmarks = label['gt_landmarks'] - gt_label['gt_landmarks'] = gt_landmarks - face_label_dict = assign_anchor_fpn(feat_shape_list, gt_label, im_info, config.FACE_LANDMARK, prefix='face') + gt_landmarks = label['gt_landmarks'] + gt_label['gt_landmarks'] = gt_landmarks + face_label_dict = assign_anchor_fpn(feat_shape_list, + gt_label, + im_info, + config.FACE_LANDMARK, + prefix='face') label_dict.update(face_label_dict) #print('im_info', im_info.shape) #print(gt_boxes.shape) for k in self.label_name: - label[k] = label_dict[k] + label[k] = label_dict[k] all_data = dict() for key in self.data_name: @@ -472,7 +543,7 @@ class CropLoader2(mx.io.DataIter): for key in self.label_name: pad = 0 if key.startswith('bbox_') else -1 #print('label vstack', key, pad, len(label_list), file=sys.stderr) - all_label[key] = tensor_vstack([batch[key] for batch in label_list], pad=pad) + all_label[key] = tensor_vstack( + [batch[key] for batch in label_list], pad=pad) self.data = [mx.nd.array(all_data[key]) for key in self.data_name] self.label = [mx.nd.array(all_label[key]) for key in self.label_name] - diff --git a/RetinaFace/rcnn/core/metric.py b/detection/RetinaFace/rcnn/core/metric.py similarity index 54% rename from RetinaFace/rcnn/core/metric.py rename to detection/RetinaFace/rcnn/core/metric.py index d600b20..afdc925 100644 --- a/RetinaFace/rcnn/core/metric.py +++ b/detection/RetinaFace/rcnn/core/metric.py @@ -12,13 +12,12 @@ def get_rpn_names(): return pred, label - class RPNAccMetric(mx.metric.EvalMetric): - def __init__(self, pred_idx=-1, label_idx=-1,name='RPNAcc'): + def __init__(self, pred_idx=-1, label_idx=-1, name='RPNAcc'): super(RPNAccMetric, self).__init__(name) self.pred, self.label = get_rpn_names() #self.name = 'RPNAcc' - self.name = [name, name+'_BG', name+'_FG'] + self.name = [name, name + '_BG', name + '_FG'] self.pred_idx = pred_idx self.label_idx = label_idx self.STAT = [0, 0, 0] @@ -26,13 +25,12 @@ class RPNAccMetric(mx.metric.EvalMetric): def reset(self): """Clear the internal statistics to initial state.""" if isinstance(self.name, str): - self.num_inst = 0 - self.sum_metric = 0.0 + self.num_inst = 0 + self.sum_metric = 0.0 else: - #print('reset to ',len(self.name), self.name, file=sys.stderr) - self.num_inst = [0] * len(self.name) - self.sum_metric = [0.0] * len(self.name) - + #print('reset to ',len(self.name), self.name, file=sys.stderr) + self.num_inst = [0] * len(self.name) + self.sum_metric = [0.0] * len(self.name) def get(self): if isinstance(self.name, str): @@ -41,28 +39,28 @@ class RPNAccMetric(mx.metric.EvalMetric): else: return (self.name, self.sum_metric / self.num_inst) else: - names = ['%s'%(self.name[i]) for i in range(len(self.name))] + names = ['%s' % (self.name[i]) for i in range(len(self.name))] values = [x / y if y != 0 else float('nan') \ for x, y in zip(self.sum_metric, self.num_inst)] return (names, values) def update(self, labels, preds): - if self.pred_idx>=0 and self.label_idx>=0: - pred = preds[self.pred_idx] - label = preds[self.label_idx] + if self.pred_idx >= 0 and self.label_idx >= 0: + pred = preds[self.pred_idx] + label = preds[self.label_idx] else: - pred = preds[self.pred.index('rpn_cls_prob')] - label = labels[self.label.index('rpn_label')] - #label = preds[self.pred.index('rpn_label')] + pred = preds[self.pred.index('rpn_cls_prob')] + label = labels[self.label.index('rpn_label')] + #label = preds[self.pred.index('rpn_label')] num_images = pred.shape[0] #print(pred.shape, label.shape, file=sys.stderr) # pred (b, c, p) or (b, c, h, w) pred_label = mx.ndarray.argmax_channel(pred).asnumpy().astype('int32') #pred_label = pred_label.reshape((pred_label.shape[0], -1)) - pred_label = pred_label.reshape(-1,) + pred_label = pred_label.reshape(-1, ) # label (b, p) - label = label.asnumpy().astype('int32').reshape(-1,) + label = label.asnumpy().astype('int32').reshape(-1, ) #print(pred_label.shape, label.shape) # filter with keep_inds @@ -73,32 +71,32 @@ class RPNAccMetric(mx.metric.EvalMetric): _label = label[keep_inds] #print('in_metric2', pred_label.shape, label.shape, len(keep_inds), file=sys.stderr) if isinstance(self.name, str): - self.sum_metric += np.sum(_pred_label.flat == _label.flat) - self.num_inst += len(_pred_label.flat) + self.sum_metric += np.sum(_pred_label.flat == _label.flat) + self.num_inst += len(_pred_label.flat) else: - self.sum_metric[0] += np.sum(_pred_label.flat == _label.flat) - self.num_inst[0] += len(_pred_label.flat) + self.sum_metric[0] += np.sum(_pred_label.flat == _label.flat) + self.num_inst[0] += len(_pred_label.flat) - keep_inds = np.where(label == 0)[0] - _pred_label = pred_label[keep_inds] - _label = label[keep_inds] - self.sum_metric[1] += np.sum(_pred_label.flat == _label.flat) - self.num_inst[1] += len(_pred_label.flat) + keep_inds = np.where(label == 0)[0] + _pred_label = pred_label[keep_inds] + _label = label[keep_inds] + self.sum_metric[1] += np.sum(_pred_label.flat == _label.flat) + self.num_inst[1] += len(_pred_label.flat) - keep_inds = np.where(label == 1)[0] - _pred_label = pred_label[keep_inds] - _label = label[keep_inds] - a = np.sum(_pred_label.flat == _label.flat) - b = len(_pred_label.flat) - self.sum_metric[2] += a - self.num_inst[2] += b + keep_inds = np.where(label == 1)[0] + _pred_label = pred_label[keep_inds] + _label = label[keep_inds] + a = np.sum(_pred_label.flat == _label.flat) + b = len(_pred_label.flat) + self.sum_metric[2] += a + self.num_inst[2] += b - #self.STAT[0]+=a - #self.STAT[1]+=b - #self.STAT[2]+=num_images - #if self.STAT[2]%400==0: - # print('FG_ACC', self.pred_idx, self.STAT[2], self.STAT[0], self.STAT[1], float(self.STAT[0])/self.STAT[1], file=sys.stderr) - # self.STAT = [0,0,0] + #self.STAT[0]+=a + #self.STAT[1]+=b + #self.STAT[2]+=num_images + #if self.STAT[2]%400==0: + # print('FG_ACC', self.pred_idx, self.STAT[2], self.STAT[0], self.STAT[1], float(self.STAT[0])/self.STAT[1], file=sys.stderr) + # self.STAT = [0,0,0] class RPNLogLossMetric(mx.metric.EvalMetric): @@ -109,18 +107,19 @@ class RPNLogLossMetric(mx.metric.EvalMetric): self.label_idx = label_idx def update(self, labels, preds): - if self.pred_idx>=0 and self.label_idx>=0: - pred = preds[self.pred_idx] - label = preds[self.label_idx] + if self.pred_idx >= 0 and self.label_idx >= 0: + pred = preds[self.pred_idx] + label = preds[self.label_idx] else: - pred = preds[self.pred.index('rpn_cls_prob')] - label = labels[self.label.index('rpn_label')] - #label = preds[self.pred.index('rpn_label')] + pred = preds[self.pred.index('rpn_cls_prob')] + label = labels[self.label.index('rpn_label')] + #label = preds[self.pred.index('rpn_label')] # label (b, p) label = label.asnumpy().astype('int32').reshape((-1)) # pred (b, c, p) or (b, c, h, w) --> (b, p, c) --> (b*p, c) - pred = pred.asnumpy().reshape((pred.shape[0], pred.shape[1], -1)).transpose((0, 2, 1)) + pred = pred.asnumpy().reshape( + (pred.shape[0], pred.shape[1], -1)).transpose((0, 2, 1)) pred = pred.reshape((label.shape[0], -1)) # filter with keep_inds @@ -145,24 +144,23 @@ class RPNL1LossMetric(mx.metric.EvalMetric): self.name = name def update(self, labels, preds): - if self.loss_idx>=0 and self.weight_idx>=0: - bbox_loss = preds[self.loss_idx].asnumpy() - bbox_weight = preds[self.weight_idx].asnumpy() + if self.loss_idx >= 0 and self.weight_idx >= 0: + bbox_loss = preds[self.loss_idx].asnumpy() + bbox_weight = preds[self.weight_idx].asnumpy() else: - bbox_loss = preds[self.pred.index('rpn_bbox_loss')].asnumpy() - bbox_weight = labels[self.label.index('rpn_bbox_weight')].asnumpy() - #bbox_weight = preds[self.pred.index('rpn_bbox_weight')].asnumpy() + bbox_loss = preds[self.pred.index('rpn_bbox_loss')].asnumpy() + bbox_weight = labels[self.label.index('rpn_bbox_weight')].asnumpy() + #bbox_weight = preds[self.pred.index('rpn_bbox_weight')].asnumpy() #print('in_metric', self.name, bbox_weight.shape, bbox_loss.shape) # calculate num_inst (average on those fg anchors) - if config.LR_MODE==0: - num_inst = np.sum(bbox_weight > 0) / (bbox_weight.shape[1]/config.NUM_ANCHORS) + if config.LR_MODE == 0: + num_inst = np.sum(bbox_weight > 0) / (bbox_weight.shape[1] / + config.NUM_ANCHORS) else: - num_inst = 1 + num_inst = 1 #print('in_metric log', bbox_loss.shape, num_inst, file=sys.stderr) self.sum_metric += np.sum(bbox_loss) self.num_inst += num_inst - - diff --git a/RetinaFace/rcnn/core/module.py b/detection/RetinaFace/rcnn/core/module.py similarity index 69% rename from RetinaFace/rcnn/core/module.py rename to detection/RetinaFace/rcnn/core/module.py index bf28f8e..731a4ff 100644 --- a/RetinaFace/rcnn/core/module.py +++ b/detection/RetinaFace/rcnn/core/module.py @@ -10,6 +10,7 @@ from mxnet.initializer import Uniform from mxnet.module.base_module import BaseModule from mxnet.module.module import Module + class MutableModule(BaseModule): """A mutable module is a module that supports variable input data. @@ -25,9 +26,16 @@ class MutableModule(BaseModule): max_label_shapes : list of (name, shape) tuple, designating inputs whose shape vary fixed_param_prefix : list of str, indicating fixed parameters """ - def __init__(self, symbol, data_names, label_names, - logger=logging, context=ctx.cpu(), work_load_list=None, - max_data_shapes=None, max_label_shapes=None, fixed_param_prefix=None): + def __init__(self, + symbol, + data_names, + label_names, + logger=logging, + context=ctx.cpu(), + work_load_list=None, + max_data_shapes=None, + max_label_shapes=None, + fixed_param_prefix=None): super(MutableModule, self).__init__(logger=logger) self._symbol = symbol self._data_names = data_names @@ -79,18 +87,31 @@ class MutableModule(BaseModule): assert self.binded and self.params_initialized return self._curr_module.get_params() - def init_params(self, initializer=Uniform(0.01), arg_params=None, aux_params=None, - allow_missing=False, force_init=False, allow_extra=False): + def init_params(self, + initializer=Uniform(0.01), + arg_params=None, + aux_params=None, + allow_missing=False, + force_init=False, + allow_extra=False): if self.params_initialized and not force_init: return assert self.binded, 'call bind before initializing the parameters' - self._curr_module.init_params(initializer=initializer, arg_params=arg_params, - aux_params=aux_params, allow_missing=allow_missing, - force_init=force_init, allow_extra=allow_extra) + self._curr_module.init_params(initializer=initializer, + arg_params=arg_params, + aux_params=aux_params, + allow_missing=allow_missing, + force_init=force_init, + allow_extra=allow_extra) self.params_initialized = True - def bind(self, data_shapes, label_shapes=None, for_training=True, - inputs_need_grad=False, force_rebind=False, shared_module=None): + def bind(self, + data_shapes, + label_shapes=None, + for_training=True, + inputs_need_grad=False, + force_rebind=False, + shared_module=None): # in case we already initialized params, keep it if self.params_initialized: arg_params, aux_params = self.get_params() @@ -134,25 +155,38 @@ class MutableModule(BaseModule): if len(max_label_shapes) == 0: max_label_shapes = None - module = Module(self._symbol, self._data_names, self._label_names, logger=self.logger, - context=self._context, work_load_list=self._work_load_list, + module = Module(self._symbol, + self._data_names, + self._label_names, + logger=self.logger, + context=self._context, + work_load_list=self._work_load_list, fixed_param_names=self._fixed_param_names) - module.bind(max_data_shapes, max_label_shapes, for_training, inputs_need_grad, - force_rebind=False, shared_module=None) + module.bind(max_data_shapes, + max_label_shapes, + for_training, + inputs_need_grad, + force_rebind=False, + shared_module=None) self._curr_module = module # copy back saved params, if already initialized if self.params_initialized: self.set_params(arg_params, aux_params) - def init_optimizer(self, kvstore='local', optimizer='sgd', - optimizer_params=(('learning_rate', 0.01),), force_init=False): + def init_optimizer(self, + kvstore='local', + optimizer='sgd', + optimizer_params=(('learning_rate', 0.01), ), + force_init=False): assert self.binded and self.params_initialized if self.optimizer_initialized and not force_init: self.logger.warning('optimizer already initialized, ignoring.') return - self._curr_module.init_optimizer(kvstore, optimizer, optimizer_params, + self._curr_module.init_optimizer(kvstore, + optimizer, + optimizer_params, force_init=force_init) self.optimizer_initialized = True @@ -161,13 +195,15 @@ class MutableModule(BaseModule): # get current_shapes if self._curr_module.label_shapes is not None: - current_shapes = dict(self._curr_module.data_shapes + self._curr_module.label_shapes) + current_shapes = dict(self._curr_module.data_shapes + + self._curr_module.label_shapes) else: current_shapes = dict(self._curr_module.data_shapes) # get input_shapes if data_batch.provide_label is not None: - input_shapes = dict(data_batch.provide_data + data_batch.provide_label) + input_shapes = dict(data_batch.provide_data + + data_batch.provide_label) else: input_shapes = dict(data_batch.provide_data) @@ -178,12 +214,18 @@ class MutableModule(BaseModule): shape_changed = True if shape_changed: - module = Module(self._symbol, self._data_names, self._label_names, - logger=self.logger, context=self._context, + module = Module(self._symbol, + self._data_names, + self._label_names, + logger=self.logger, + context=self._context, work_load_list=self._work_load_list, fixed_param_names=self._fixed_param_names) - module.bind(data_batch.provide_data, data_batch.provide_label, self._curr_module.for_training, - self._curr_module.inputs_need_grad, force_rebind=False, + module.bind(data_batch.provide_data, + data_batch.provide_label, + self._curr_module.for_training, + self._curr_module.inputs_need_grad, + force_rebind=False, shared_module=self._curr_module) self._curr_module = module @@ -199,11 +241,13 @@ class MutableModule(BaseModule): def get_outputs(self, merge_multi_context=True): assert self.binded and self.params_initialized - return self._curr_module.get_outputs(merge_multi_context=merge_multi_context) + return self._curr_module.get_outputs( + merge_multi_context=merge_multi_context) def get_input_grads(self, merge_multi_context=True): assert self.binded and self.params_initialized and self.inputs_need_grad - return self._curr_module.get_input_grads(merge_multi_context=merge_multi_context) + return self._curr_module.get_input_grads( + merge_multi_context=merge_multi_context) def update_metric(self, eval_metric, labels): assert self.binded and self.params_initialized diff --git a/RetinaFace/rcnn/core/module_bak.py b/detection/RetinaFace/rcnn/core/module_bak.py similarity index 69% rename from RetinaFace/rcnn/core/module_bak.py rename to detection/RetinaFace/rcnn/core/module_bak.py index 569d50d..2c81902 100644 --- a/RetinaFace/rcnn/core/module_bak.py +++ b/detection/RetinaFace/rcnn/core/module_bak.py @@ -10,6 +10,7 @@ from mxnet.initializer import Uniform from mxnet.module.base_module import BaseModule from mxnet.module.module import Module + class MutableModule(BaseModule): """A mutable module is a module that supports variable input data. @@ -25,9 +26,16 @@ class MutableModule(BaseModule): max_label_shapes : list of (name, shape) tuple, designating inputs whose shape vary fixed_param_prefix : list of str, indicating fixed parameters """ - def __init__(self, symbol, data_names, label_names, - logger=logging, context=ctx.cpu(), work_load_list=None, - max_data_shapes=None, max_label_shapes=None, fixed_param_prefix=None): + def __init__(self, + symbol, + data_names, + label_names, + logger=logging, + context=ctx.cpu(), + work_load_list=None, + max_data_shapes=None, + max_label_shapes=None, + fixed_param_prefix=None): super(MutableModule, self).__init__(logger=logger) self._symbol = symbol self._data_names = data_names @@ -79,18 +87,32 @@ class MutableModule(BaseModule): assert self.binded and self.params_initialized return self._curr_module.get_params() - def init_params(self, initializer=Uniform(0.01), arg_params=None, aux_params=None, - allow_missing=False, force_init=False, allow_extra=False): + def init_params(self, + initializer=Uniform(0.01), + arg_params=None, + aux_params=None, + allow_missing=False, + force_init=False, + allow_extra=False): if self.params_initialized and not force_init: return assert self.binded, 'call bind before initializing the parameters' - self._curr_module.init_params(initializer=initializer, arg_params=arg_params, - aux_params=aux_params, allow_missing=allow_missing, - force_init=force_init, allow_extra=allow_extra) + self._curr_module.init_params(initializer=initializer, + arg_params=arg_params, + aux_params=aux_params, + allow_missing=allow_missing, + force_init=force_init, + allow_extra=allow_extra) self.params_initialized = True - def bind(self, data_shapes, label_shapes=None, for_training=True, - inputs_need_grad=False, force_rebind=False, shared_module=None, grad_req='write'): + def bind(self, + data_shapes, + label_shapes=None, + for_training=True, + inputs_need_grad=False, + force_rebind=False, + shared_module=None, + grad_req='write'): # in case we already initialized params, keep it if self.params_initialized: arg_params, aux_params = self.get_params() @@ -134,25 +156,38 @@ class MutableModule(BaseModule): if len(max_label_shapes) == 0: max_label_shapes = None - module = Module(self._symbol, self._data_names, self._label_names, logger=self.logger, - context=self._context, work_load_list=self._work_load_list, + module = Module(self._symbol, + self._data_names, + self._label_names, + logger=self.logger, + context=self._context, + work_load_list=self._work_load_list, fixed_param_names=self._fixed_param_names) - module.bind(max_data_shapes, max_label_shapes, for_training, inputs_need_grad, - force_rebind=False, shared_module=None) + module.bind(max_data_shapes, + max_label_shapes, + for_training, + inputs_need_grad, + force_rebind=False, + shared_module=None) self._curr_module = module # copy back saved params, if already initialized if self.params_initialized: self.set_params(arg_params, aux_params) - def init_optimizer(self, kvstore='local', optimizer='sgd', - optimizer_params=(('learning_rate', 0.01),), force_init=False): + def init_optimizer(self, + kvstore='local', + optimizer='sgd', + optimizer_params=(('learning_rate', 0.01), ), + force_init=False): assert self.binded and self.params_initialized if self.optimizer_initialized and not force_init: self.logger.warning('optimizer already initialized, ignoring.') return - self._curr_module.init_optimizer(kvstore, optimizer, optimizer_params, + self._curr_module.init_optimizer(kvstore, + optimizer, + optimizer_params, force_init=force_init) self.optimizer_initialized = True @@ -161,13 +196,15 @@ class MutableModule(BaseModule): # get current_shapes if self._curr_module.label_shapes is not None: - current_shapes = dict(self._curr_module.data_shapes + self._curr_module.label_shapes) + current_shapes = dict(self._curr_module.data_shapes + + self._curr_module.label_shapes) else: current_shapes = dict(self._curr_module.data_shapes) # get input_shapes if data_batch.provide_label is not None: - input_shapes = dict(data_batch.provide_data + data_batch.provide_label) + input_shapes = dict(data_batch.provide_data + + data_batch.provide_label) else: input_shapes = dict(data_batch.provide_data) @@ -178,12 +215,18 @@ class MutableModule(BaseModule): shape_changed = True if shape_changed: - module = Module(self._symbol, self._data_names, self._label_names, - logger=self.logger, context=self._context, + module = Module(self._symbol, + self._data_names, + self._label_names, + logger=self.logger, + context=self._context, work_load_list=self._work_load_list, fixed_param_names=self._fixed_param_names) - module.bind(data_batch.provide_data, data_batch.provide_label, self._curr_module.for_training, - self._curr_module.inputs_need_grad, force_rebind=False, + module.bind(data_batch.provide_data, + data_batch.provide_label, + self._curr_module.for_training, + self._curr_module.inputs_need_grad, + force_rebind=False, shared_module=self._curr_module) self._curr_module = module @@ -199,11 +242,13 @@ class MutableModule(BaseModule): def get_outputs(self, merge_multi_context=True): assert self.binded and self.params_initialized - return self._curr_module.get_outputs(merge_multi_context=merge_multi_context) + return self._curr_module.get_outputs( + merge_multi_context=merge_multi_context) def get_input_grads(self, merge_multi_context=True): assert self.binded and self.params_initialized and self.inputs_need_grad - return self._curr_module.get_input_grads(merge_multi_context=merge_multi_context) + return self._curr_module.get_input_grads( + merge_multi_context=merge_multi_context) def update_metric(self, eval_metric, labels): assert self.binded and self.params_initialized diff --git a/RetinaFace/rcnn/core/tester.py b/detection/RetinaFace/rcnn/core/tester.py similarity index 73% rename from RetinaFace/rcnn/core/tester.py rename to detection/RetinaFace/rcnn/core/tester.py index 8d981fa..d9d5d32 100644 --- a/RetinaFace/rcnn/core/tester.py +++ b/detection/RetinaFace/rcnn/core/tester.py @@ -20,40 +20,46 @@ from rcnn.processing.nms import py_nms_wrapper, cpu_nms_wrapper, gpu_nms_wrapper from rcnn.processing.bbox_transform import bbox_overlaps +def IOU(Reframe, GTframe): + x1 = Reframe[0] + y1 = Reframe[1] + width1 = Reframe[2] - Reframe[0] + height1 = Reframe[3] - Reframe[1] -def IOU(Reframe,GTframe): - x1 = Reframe[0]; - y1 = Reframe[1]; - width1 = Reframe[2]-Reframe[0]; - height1 = Reframe[3]-Reframe[1]; + x2 = GTframe[0] + y2 = GTframe[1] + width2 = GTframe[2] - GTframe[0] + height2 = GTframe[3] - GTframe[1] - x2 = GTframe[0] - y2 = GTframe[1] - width2 = GTframe[2]-GTframe[0] - height2 = GTframe[3]-GTframe[1] + endx = max(x1 + width1, x2 + width2) + startx = min(x1, x2) + width = width1 + width2 - (endx - startx) - endx = max(x1+width1,x2+width2) - startx = min(x1,x2) - width = width1+width2-(endx-startx) + endy = max(y1 + height1, y2 + height2) + starty = min(y1, y2) + height = height1 + height2 - (endy - starty) - endy = max(y1+height1,y2+height2) - starty = min(y1,y2) - height = height1+height2-(endy-starty) + if width <= 0 or height <= 0: + ratio = 0 + else: + Area = width * height + Area1 = width1 * height1 + Area2 = width2 * height2 + ratio = Area * 1. / (Area1 + Area2 - Area) + return ratio - if width <=0 or height <= 0: - ratio = 0 - else: - Area = width*height - Area1 = width1*height1 - Area2 = width2*height2 - ratio = Area*1./(Area1+Area2-Area) - return ratio class Predictor(object): - def __init__(self, symbol, data_names, label_names, - context=mx.cpu(), max_data_shapes=None, - provide_data=None, provide_label=None, - arg_params=None, aux_params=None): + def __init__(self, + symbol, + data_names, + label_names, + context=mx.cpu(), + max_data_shapes=None, + provide_data=None, + provide_label=None, + arg_params=None, + aux_params=None): #self._mod = MutableModule(symbol, data_names, label_names, # context=context, max_data_shapes=max_data_shapes) self._mod = Module(symbol, data_names, label_names, context=context) @@ -62,7 +68,8 @@ class Predictor(object): def predict(self, data_batch): self._mod.forward(data_batch) - return dict(zip(self._mod.output_names, self._mod.get_outputs())) #TODO + return dict(zip(self._mod.output_names, + self._mod.get_outputs())) #TODO #return self._mod.get_outputs() @@ -79,6 +86,7 @@ def im_proposal(predictor, data_batch, data_names, scale): return scores, boxes, data_dict + def _im_proposal(predictor, data_batch, data_names, scale): data_dict = dict(zip(data_names, data_batch.data)) output = predictor.predict(data_batch) @@ -116,7 +124,8 @@ def generate_proposals(predictor, test_data, imdb, vis=False, thresh=0.): t = time.time() scale = im_info[0, 2] - scores, boxes, data_dict = im_proposal(predictor, data_batch, data_names, scale) + scores, boxes, data_dict = im_proposal(predictor, data_batch, + data_names, scale) print(scores.shape, boxes.shape, file=sys.stderr) t2 = time.time() - t t = time.time() @@ -131,11 +140,12 @@ def generate_proposals(predictor, test_data, imdb, vis=False, thresh=0.): imdb_boxes.append(dets) if vis: - vis_all_detection(data_dict['data'].asnumpy(), [dets], ['obj'], scale) + vis_all_detection(data_dict['data'].asnumpy(), [dets], ['obj'], + scale) logger.info('generating %d/%d ' % (i + 1, imdb.num_images) + - 'proposal %d ' % (dets.shape[0]) + - 'data %.4fs net %.4fs' % (t1, t2)) + 'proposal %d ' % (dets.shape[0]) + 'data %.4fs net %.4fs' % + (t1, t2)) i += 1 assert len(imdb_boxes) == imdb.num_images, 'calculations not complete' @@ -157,6 +167,7 @@ def generate_proposals(predictor, test_data, imdb, vis=False, thresh=0.): logger.info('wrote rpn proposals to %s' % rpn_file) return imdb_boxes + def test_proposals(predictor, test_data, imdb, roidb, vis=False): """ Test detections results using RPN. @@ -182,7 +193,7 @@ def test_proposals(predictor, test_data, imdb, roidb, vis=False): original_boxes = list() gt_overlaps = np.zeros(0) overall = [0.0, 0.0] - gt_max = np.array( (0.0, 0.0) ) + gt_max = np.array((0.0, 0.0)) num_pos = 0 #apply scale, for SSH #_, roidb = image.get_image(roidb) @@ -192,8 +203,9 @@ def test_proposals(predictor, test_data, imdb, roidb, vis=False): oscale = im_info[0, 2] #print('scale', scale, file=sys.stderr) - scale = 1.0 #fix scale=1.0 for SSH face detector - scores, boxes, data_dict = im_proposal(predictor, data_batch, data_names, scale) + scale = 1.0 #fix scale=1.0 for SSH face detector + scores, boxes, data_dict = im_proposal(predictor, data_batch, + data_names, scale) #print(scores.shape, boxes.shape, file=sys.stderr) t2 = time.time() - t t = time.time() @@ -207,24 +219,27 @@ def test_proposals(predictor, test_data, imdb, roidb, vis=False): dets = dets[keep, :] imdb_boxes.append(dets) - logger.info('generating %d/%d ' % (i + 1, imdb.num_images) + - 'proposal %d ' % (dets.shape[0]) + - 'data %.4fs net %.4fs' % (t1, t2)) + 'proposal %d ' % (dets.shape[0]) + 'data %.4fs net %.4fs' % + (t1, t2)) #if dets.shape[0]==0: # continue if vis: - vis_all_detection(data_dict['data'].asnumpy(), [dets], ['obj'], scale) + vis_all_detection(data_dict['data'].asnumpy(), [dets], ['obj'], + scale) boxes = dets #max_gt_overlaps = roidb[i]['gt_overlaps'].max(axis=1) #gt_inds = np.where((roidb[i]['gt_classes'] > 0) & (max_gt_overlaps == 1))[0] #gt_boxes = roidb[i]['boxes'][gt_inds, :] - gt_boxes = roidb[i]['boxes'].copy() * oscale # as roidb is the original one, need to scale GT for SSH - gt_areas = (gt_boxes[:, 2] - gt_boxes[:, 0] + 1) * (gt_boxes[:, 3] - gt_boxes[:, 1] + 1) + gt_boxes = roidb[i]['boxes'].copy( + ) * oscale # as roidb is the original one, need to scale GT for SSH + gt_areas = (gt_boxes[:, 2] - gt_boxes[:, 0] + 1) * (gt_boxes[:, 3] - + gt_boxes[:, 1] + 1) num_pos += gt_boxes.shape[0] - overlaps = bbox_overlaps(boxes.astype(np.float), gt_boxes.astype(np.float)) + overlaps = bbox_overlaps(boxes.astype(np.float), + gt_boxes.astype(np.float)) #print(im_info, gt_boxes.shape, boxes.shape, overlaps.shape, file=sys.stderr) _gt_overlaps = np.zeros((gt_boxes.shape[0])) @@ -259,44 +274,55 @@ def test_proposals(predictor, test_data, imdb, roidb, vis=False): # overlaps[box_ind, :] = -1 # overlaps[:, gt_ind] = -1 - if boxes.shape[0]>0: - _gt_overlaps = overlaps.max(axis=0) - #print('max_overlaps', _gt_overlaps, file=sys.stderr) - for j in range(len(_gt_overlaps)): - if _gt_overlaps[j]>config.TEST.IOU_THRESH: - continue - print(j, 'failed', gt_boxes[j], 'max_overlap:', _gt_overlaps[j], file=sys.stderr) - #_idx = np.where(overlaps[:,j]>0.4)[0] - #print(j, _idx, file=sys.stderr) - #print(overlaps[_idx,j], file=sys.stderr) - #for __idx in _idx: - # print(gt_boxes[j], boxes[__idx], overlaps[__idx,j], IOU(gt_boxes[j], boxes[__idx,0:4]), file=sys.stderr) + if boxes.shape[0] > 0: + _gt_overlaps = overlaps.max(axis=0) + #print('max_overlaps', _gt_overlaps, file=sys.stderr) + for j in range(len(_gt_overlaps)): + if _gt_overlaps[j] > config.TEST.IOU_THRESH: + continue + print(j, + 'failed', + gt_boxes[j], + 'max_overlap:', + _gt_overlaps[j], + file=sys.stderr) + #_idx = np.where(overlaps[:,j]>0.4)[0] + #print(j, _idx, file=sys.stderr) + #print(overlaps[_idx,j], file=sys.stderr) + #for __idx in _idx: + # print(gt_boxes[j], boxes[__idx], overlaps[__idx,j], IOU(gt_boxes[j], boxes[__idx,0:4]), file=sys.stderr) - # append recorded IoU coverage level - found = (_gt_overlaps > config.TEST.IOU_THRESH).sum() - _recall = found / float(gt_boxes.shape[0]) - print('recall', _recall, gt_boxes.shape[0], boxes.shape[0], gt_areas, file=sys.stderr) - overall[0]+=found - overall[1]+=gt_boxes.shape[0] - #gt_overlaps = np.hstack((gt_overlaps, _gt_overlaps)) - #_recall = (gt_overlaps >= threshold).sum() / float(num_pos) - _recall = float(overall[0])/overall[1] - print('recall_all', _recall, file=sys.stderr) + # append recorded IoU coverage level + found = (_gt_overlaps > config.TEST.IOU_THRESH).sum() + _recall = found / float(gt_boxes.shape[0]) + print('recall', + _recall, + gt_boxes.shape[0], + boxes.shape[0], + gt_areas, + file=sys.stderr) + overall[0] += found + overall[1] += gt_boxes.shape[0] + #gt_overlaps = np.hstack((gt_overlaps, _gt_overlaps)) + #_recall = (gt_overlaps >= threshold).sum() / float(num_pos) + _recall = float(overall[0]) / overall[1] + print('recall_all', _recall, file=sys.stderr) - - boxes[:,0:4] /= oscale + boxes[:, 0:4] /= oscale _vec = roidb[i]['image'].split('/') out_dir = os.path.join(output_folder, _vec[-2]) if not os.path.exists(out_dir): os.mkdir(out_dir) out_file = os.path.join(out_dir, _vec[-1].replace('jpg', 'txt')) with open(out_file, 'w') as f: - name = '/'.join(roidb[i]['image'].split('/')[-2:]) - f.write("%s\n"%(name)) - f.write("%d\n"%(boxes.shape[0])) - for b in range(boxes.shape[0]): - box = boxes[b] - f.write("%d %d %d %d %g \n"%(box[0], box[1], box[2]-box[0], box[3]-box[1], box[4])) + name = '/'.join(roidb[i]['image'].split('/')[-2:]) + f.write("%s\n" % (name)) + f.write("%d\n" % (boxes.shape[0])) + for b in range(boxes.shape[0]): + box = boxes[b] + f.write( + "%d %d %d %d %g \n" % + (box[0], box[1], box[2] - box[0], box[3] - box[1], box[4])) i += 1 #bbox_f.close() @@ -314,9 +340,6 @@ def test_proposals(predictor, test_data, imdb, roidb, vis=False): for threshold, recall in zip(thresholds, recalls): print('recall @{:.2f}: {:.3f}'.format(threshold, recall)) - - - assert len(imdb_boxes) == imdb.num_images, 'calculations not complete' # save results @@ -328,6 +351,7 @@ def test_proposals(predictor, test_data, imdb, roidb, vis=False): logger.info('wrote rpn proposals to %s' % rpn_file) return imdb_boxes + def im_detect(predictor, data_batch, data_names, scale): output = predictor.predict(data_batch) @@ -385,7 +409,8 @@ def pred_eval(predictor, test_data, imdb, vis=False, thresh=1e-3): t = time.time() scale = im_info[0, 2] - scores, boxes, data_dict = im_detect(predictor, data_batch, data_names, scale) + scores, boxes, data_dict = im_detect(predictor, data_batch, data_names, + scale) t2 = time.time() - t t = time.time() @@ -399,8 +424,8 @@ def pred_eval(predictor, test_data, imdb, vis=False, thresh=1e-3): all_boxes[j][i] = cls_dets[keep, :] if max_per_image > 0: - image_scores = np.hstack([all_boxes[j][i][:, -1] - for j in range(1, imdb.num_classes)]) + image_scores = np.hstack( + [all_boxes[j][i][:, -1] for j in range(1, imdb.num_classes)]) if len(image_scores) > max_per_image: image_thresh = np.sort(image_scores)[-max_per_image] for j in range(1, imdb.num_classes): @@ -408,12 +433,16 @@ def pred_eval(predictor, test_data, imdb, vis=False, thresh=1e-3): all_boxes[j][i] = all_boxes[j][i][keep, :] if vis: - boxes_this_image = [[]] + [all_boxes[j][i] for j in range(1, imdb.num_classes)] - vis_all_detection(data_dict['data'].asnumpy(), boxes_this_image, imdb.classes, scale) + boxes_this_image = [[]] + [ + all_boxes[j][i] for j in range(1, imdb.num_classes) + ] + vis_all_detection(data_dict['data'].asnumpy(), boxes_this_image, + imdb.classes, scale) t3 = time.time() - t t = time.time() - logger.info('testing %d/%d data %.4fs net %.4fs post %.4fs' % (i, imdb.num_images, t1, t2, t3)) + logger.info('testing %d/%d data %.4fs net %.4fs post %.4fs' % + (i, imdb.num_images, t1, t2, t3)) i += 1 det_file = os.path.join(imdb.cache_path, imdb.name + '_detections.pkl') @@ -439,19 +468,25 @@ def vis_all_detection(im_array, detections, class_names, scale): for j, name in enumerate(class_names): if name == '__background__': continue - color = (random.random(), random.random(), random.random()) # generate a random color + color = (random.random(), random.random(), random.random() + ) # generate a random color dets = detections[j] for det in dets: bbox = det[:4] * scale score = det[-1] rect = plt.Rectangle((bbox[0], bbox[1]), bbox[2] - bbox[0], - bbox[3] - bbox[1], fill=False, - edgecolor=color, linewidth=3.5) + bbox[3] - bbox[1], + fill=False, + edgecolor=color, + linewidth=3.5) plt.gca().add_patch(rect) - plt.gca().text(bbox[0], bbox[1] - 2, + plt.gca().text(bbox[0], + bbox[1] - 2, '{:s} {:.3f}'.format(name, score), - bbox=dict(facecolor=color, alpha=0.5), fontsize=12, color='white') + bbox=dict(facecolor=color, alpha=0.5), + fontsize=12, + color='white') plt.show() @@ -473,13 +508,20 @@ def draw_all_detection(im_array, detections, class_names, scale): for j, name in enumerate(class_names): if name == '__background__': continue - color = (random.randint(0, 256), random.randint(0, 256), random.randint(0, 256)) # generate a random color + color = (random.randint(0, 256), random.randint(0, 256), + random.randint(0, 256)) # generate a random color dets = detections[j] for det in dets: bbox = det[:4] * scale score = det[-1] bbox = map(int, bbox) - cv2.rectangle(im, (bbox[0], bbox[1]), (bbox[2], bbox[3]), color=color, thickness=2) - cv2.putText(im, '%s %.3f' % (class_names[j], score), (bbox[0], bbox[1] + 10), - color=color_white, fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.5) + cv2.rectangle(im, (bbox[0], bbox[1]), (bbox[2], bbox[3]), + color=color, + thickness=2) + cv2.putText(im, + '%s %.3f' % (class_names[j], score), + (bbox[0], bbox[1] + 10), + color=color_white, + fontFace=cv2.FONT_HERSHEY_COMPLEX, + fontScale=0.5) return im diff --git a/RetinaFace/rcnn/cython/.gitignore b/detection/RetinaFace/rcnn/cython/.gitignore similarity index 100% rename from RetinaFace/rcnn/cython/.gitignore rename to detection/RetinaFace/rcnn/cython/.gitignore diff --git a/RetinaFace/rcnn/cython/__init__.py b/detection/RetinaFace/rcnn/cython/__init__.py similarity index 100% rename from RetinaFace/rcnn/cython/__init__.py rename to detection/RetinaFace/rcnn/cython/__init__.py diff --git a/RetinaFace/rcnn/cython/anchors.pyx b/detection/RetinaFace/rcnn/cython/anchors.pyx similarity index 100% rename from RetinaFace/rcnn/cython/anchors.pyx rename to detection/RetinaFace/rcnn/cython/anchors.pyx diff --git a/RetinaFace/rcnn/cython/bbox.pyx b/detection/RetinaFace/rcnn/cython/bbox.pyx similarity index 100% rename from RetinaFace/rcnn/cython/bbox.pyx rename to detection/RetinaFace/rcnn/cython/bbox.pyx diff --git a/RetinaFace/rcnn/cython/cpu_nms.pyx b/detection/RetinaFace/rcnn/cython/cpu_nms.pyx similarity index 100% rename from RetinaFace/rcnn/cython/cpu_nms.pyx rename to detection/RetinaFace/rcnn/cython/cpu_nms.pyx diff --git a/RetinaFace/rcnn/cython/gpu_nms.hpp b/detection/RetinaFace/rcnn/cython/gpu_nms.hpp similarity index 100% rename from RetinaFace/rcnn/cython/gpu_nms.hpp rename to detection/RetinaFace/rcnn/cython/gpu_nms.hpp diff --git a/RetinaFace/rcnn/cython/gpu_nms.pyx b/detection/RetinaFace/rcnn/cython/gpu_nms.pyx similarity index 100% rename from RetinaFace/rcnn/cython/gpu_nms.pyx rename to detection/RetinaFace/rcnn/cython/gpu_nms.pyx diff --git a/RetinaFace/rcnn/cython/nms_kernel.cu b/detection/RetinaFace/rcnn/cython/nms_kernel.cu similarity index 100% rename from RetinaFace/rcnn/cython/nms_kernel.cu rename to detection/RetinaFace/rcnn/cython/nms_kernel.cu diff --git a/RetinaFace/rcnn/cython/setup.py b/detection/RetinaFace/rcnn/cython/setup.py similarity index 75% rename from RetinaFace/rcnn/cython/setup.py rename to detection/RetinaFace/rcnn/cython/setup.py index 3e27add..1af1a1a 100644 --- a/RetinaFace/rcnn/cython/setup.py +++ b/detection/RetinaFace/rcnn/cython/setup.py @@ -41,18 +41,25 @@ def locate_cuda(): else: # otherwise, search the PATH for NVCC default_path = pjoin(os.sep, 'usr', 'local', 'cuda', 'bin') - nvcc = find_in_path('nvcc', os.environ['PATH'] + os.pathsep + default_path) + nvcc = find_in_path('nvcc', + os.environ['PATH'] + os.pathsep + default_path) if nvcc is None: - raise EnvironmentError('The nvcc binary could not be ' - 'located in your $PATH. Either add it to your path, or set $CUDAHOME') + raise EnvironmentError( + 'The nvcc binary could not be ' + 'located in your $PATH. Either add it to your path, or set $CUDAHOME' + ) home = os.path.dirname(os.path.dirname(nvcc)) - cudaconfig = {'home':home, 'nvcc':nvcc, - 'include': pjoin(home, 'include'), - 'lib64': pjoin(home, 'lib64')} + cudaconfig = { + 'home': home, + 'nvcc': nvcc, + 'include': pjoin(home, 'include'), + 'lib64': pjoin(home, 'lib64') + } for k, v in cudaconfig.items(): if not os.path.exists(v): - raise EnvironmentError('The CUDA %s path could not be located in %s' % (k, v)) + raise EnvironmentError( + 'The CUDA %s path could not be located in %s' % (k, v)) return cudaconfig @@ -63,7 +70,6 @@ try: except EnvironmentError: CUDA = None - # Obtain the numpy include directory. This logic works across numpy versions. try: numpy_include = np.get_include() @@ -117,29 +123,21 @@ class custom_build_ext(build_ext): ext_modules = [ - Extension( - "bbox", - ["bbox.pyx"], - extra_compile_args={'gcc': ["-Wno-cpp", "-Wno-unused-function"]}, - include_dirs=[numpy_include] - ), - Extension( - "anchors", - ["anchors.pyx"], - extra_compile_args={'gcc': ["-Wno-cpp", "-Wno-unused-function"]}, - include_dirs=[numpy_include] - ), - Extension( - "cpu_nms", - ["cpu_nms.pyx"], - extra_compile_args={'gcc': ["-Wno-cpp", "-Wno-unused-function"]}, - include_dirs = [numpy_include] - ), + Extension("bbox", ["bbox.pyx"], + extra_compile_args={'gcc': ["-Wno-cpp", "-Wno-unused-function"]}, + include_dirs=[numpy_include]), + Extension("anchors", ["anchors.pyx"], + extra_compile_args={'gcc': ["-Wno-cpp", "-Wno-unused-function"]}, + include_dirs=[numpy_include]), + Extension("cpu_nms", ["cpu_nms.pyx"], + extra_compile_args={'gcc': ["-Wno-cpp", "-Wno-unused-function"]}, + include_dirs=[numpy_include]), ] if CUDA is not None: ext_modules.append( - Extension('gpu_nms', + Extension( + 'gpu_nms', ['nms_kernel.cu', 'gpu_nms.pyx'], library_dirs=[CUDA['lib64']], libraries=['cudart'], @@ -148,19 +146,17 @@ if CUDA is not None: # this syntax is specific to this build system # we're only going to use certain compiler args with nvcc and not with # gcc the implementation of this trick is in customize_compiler() below - extra_compile_args={'gcc': ["-Wno-unused-function"], - 'nvcc': ['-arch=sm_35', - '--ptxas-options=-v', - '-c', - '--compiler-options', - "'-fPIC'"]}, - include_dirs = [numpy_include, CUDA['include']] - ) - ) + extra_compile_args={ + 'gcc': ["-Wno-unused-function"], + 'nvcc': [ + '-arch=sm_35', '--ptxas-options=-v', '-c', + '--compiler-options', "'-fPIC'" + ] + }, + include_dirs=[numpy_include, CUDA['include']])) else: print('Skipping GPU_NMS') - setup( name='frcnn_cython', ext_modules=ext_modules, diff --git a/RetinaFace/rcnn/dataset/__init__.py b/detection/RetinaFace/rcnn/dataset/__init__.py similarity index 100% rename from RetinaFace/rcnn/dataset/__init__.py rename to detection/RetinaFace/rcnn/dataset/__init__.py diff --git a/RetinaFace/rcnn/dataset/ds_utils.py b/detection/RetinaFace/rcnn/dataset/ds_utils.py similarity index 100% rename from RetinaFace/rcnn/dataset/ds_utils.py rename to detection/RetinaFace/rcnn/dataset/ds_utils.py diff --git a/RetinaFace/rcnn/dataset/imdb.py b/detection/RetinaFace/rcnn/dataset/imdb.py similarity index 70% rename from RetinaFace/rcnn/dataset/imdb.py rename to detection/RetinaFace/rcnn/dataset/imdb.py index d034206..3c19817 100644 --- a/RetinaFace/rcnn/dataset/imdb.py +++ b/detection/RetinaFace/rcnn/dataset/imdb.py @@ -70,10 +70,13 @@ class IMDB(object): def load_rpn_data(self, full=False): if full: - rpn_file = os.path.join(self.root_path, 'rpn_data', self.name + '_full_rpn.pkl') + rpn_file = os.path.join(self.root_path, 'rpn_data', + self.name + '_full_rpn.pkl') else: - rpn_file = os.path.join(self.root_path, 'rpn_data', self.name + '_rpn.pkl') - assert os.path.exists(rpn_file), '%s rpn data not found at %s' % (self.name, rpn_file) + rpn_file = os.path.join(self.root_path, 'rpn_data', + self.name + '_rpn.pkl') + assert os.path.exists( + rpn_file), '%s rpn data not found at %s' % (self.name, rpn_file) logger.info('%s loading rpn data from %s' % (self.name, rpn_file)) with open(rpn_file, 'rb') as f: box_list = pickle.load(f) @@ -110,7 +113,9 @@ class IMDB(object): :param gt_roidb: [image_index]['boxes', 'gt_classes', 'gt_overlaps', 'flipped'] :return: roidb: [image_index]['boxes', 'gt_classes', 'gt_overlaps', 'flipped'] """ - assert len(box_list) == self.num_images, 'number of boxes matrix must match number of images' + assert len( + box_list + ) == self.num_images, 'number of boxes matrix must match number of images' roidb = [] for i in range(self.num_images): roi_rec = dict() @@ -122,24 +127,34 @@ class IMDB(object): if boxes.shape[1] == 5: boxes = boxes[:, :4] num_boxes = boxes.shape[0] - overlaps = np.zeros((num_boxes, self.num_classes), dtype=np.float32) + overlaps = np.zeros((num_boxes, self.num_classes), + dtype=np.float32) if gt_roidb is not None and gt_roidb[i]['boxes'].size > 0: gt_boxes = gt_roidb[i]['boxes'] gt_classes = gt_roidb[i]['gt_classes'] # n boxes and k gt_boxes => n * k overlap - gt_overlaps = bbox_overlaps(boxes.astype(np.float), gt_boxes.astype(np.float)) + gt_overlaps = bbox_overlaps(boxes.astype(np.float), + gt_boxes.astype(np.float)) # for each box in n boxes, select only maximum overlap (must be greater than zero) argmaxes = gt_overlaps.argmax(axis=1) maxes = gt_overlaps.max(axis=1) I = np.where(maxes > 0)[0] overlaps[I, gt_classes[argmaxes[I]]] = maxes[I] - roi_rec.update({'boxes': boxes, - 'gt_classes': np.zeros((num_boxes,), dtype=np.int32), - 'gt_overlaps': overlaps, - 'max_classes': overlaps.argmax(axis=1), - 'max_overlaps': overlaps.max(axis=1), - 'flipped': False}) + roi_rec.update({ + 'boxes': + boxes, + 'gt_classes': + np.zeros((num_boxes, ), dtype=np.int32), + 'gt_overlaps': + overlaps, + 'max_classes': + overlaps.argmax(axis=1), + 'max_overlaps': + overlaps.max(axis=1), + 'flipped': + False + }) # background roi => background class zero_indexes = np.where(roi_rec['max_overlaps'] == 0)[0] @@ -163,42 +178,44 @@ class IMDB(object): assert self.num_images == len(roidb) for i in range(self.num_images): roi_rec = roidb[i] - entry = {'image': roi_rec['image'], - 'stream': roi_rec['stream'], - 'height': roi_rec['height'], - 'width': roi_rec['width'], - #'boxes': boxes, - 'gt_classes': roidb[i]['gt_classes'], - 'gt_overlaps': roidb[i]['gt_overlaps'], - 'max_classes': roidb[i]['max_classes'], - 'max_overlaps': roidb[i]['max_overlaps'], - 'flipped': True} + entry = { + 'image': roi_rec['image'], + 'stream': roi_rec['stream'], + 'height': roi_rec['height'], + 'width': roi_rec['width'], + #'boxes': boxes, + 'gt_classes': roidb[i]['gt_classes'], + 'gt_overlaps': roidb[i]['gt_overlaps'], + 'max_classes': roidb[i]['max_classes'], + 'max_overlaps': roidb[i]['max_overlaps'], + 'flipped': True + } for k in roi_rec: - if not k.startswith('boxes'): - continue - boxes = roi_rec[k].copy() - oldx1 = boxes[:, 0].copy() - oldx2 = boxes[:, 2].copy() - boxes[:, 0] = roi_rec['width'] - oldx2 - 1 - boxes[:, 2] = roi_rec['width'] - oldx1 - 1 - assert (boxes[:, 2] >= boxes[:, 0]).all() - entry[k] = boxes + if not k.startswith('boxes'): + continue + boxes = roi_rec[k].copy() + oldx1 = boxes[:, 0].copy() + oldx2 = boxes[:, 2].copy() + boxes[:, 0] = roi_rec['width'] - oldx2 - 1 + boxes[:, 2] = roi_rec['width'] - oldx1 - 1 + assert (boxes[:, 2] >= boxes[:, 0]).all() + entry[k] = boxes if 'landmarks' in roi_rec: - k = 'landmarks' - landmarks = roi_rec[k].copy() - landmarks[:,:,0] *= -1 - landmarks[:,:,0] += (roi_rec['width']-1) - #for a in range(0,10,2): - # oldx1 = landmarks[:, a].copy() - # landmarks[:,a] = roi_rec['width'] - oldx1 - 1 - order = [1,0,2,4,3] - flandmarks = landmarks.copy() - for idx, a in enumerate(order): - flandmarks[:, idx,:] = landmarks[:,a,:] + k = 'landmarks' + landmarks = roi_rec[k].copy() + landmarks[:, :, 0] *= -1 + landmarks[:, :, 0] += (roi_rec['width'] - 1) + #for a in range(0,10,2): + # oldx1 = landmarks[:, a].copy() + # landmarks[:,a] = roi_rec['width'] - oldx1 - 1 + order = [1, 0, 2, 4, 3] + flandmarks = landmarks.copy() + for idx, a in enumerate(order): + flandmarks[:, idx, :] = landmarks[:, a, :] - entry[k] = flandmarks + entry[k] = flandmarks if 'blur' in roi_rec: - entry['blur'] = roi_rec['blur'] + entry['blur'] = roi_rec['blur'] roidb.append(entry) self.image_set_index *= 2 @@ -215,10 +232,12 @@ class IMDB(object): ar: average recall, recalls: vector recalls at each IoU overlap threshold thresholds: vector of IoU overlap threshold, gt_overlaps: vector of all ground-truth overlaps """ - area_names = ['all', '0-25', '25-50', '50-100', - '100-200', '200-300', '300-inf'] - area_ranges = [[0**2, 1e5**2], [0**2, 25**2], [25**2, 50**2], [50**2, 100**2], - [100**2, 200**2], [200**2, 300**2], [300**2, 1e5**2]] + area_names = [ + 'all', '0-25', '25-50', '50-100', '100-200', '200-300', '300-inf' + ] + area_ranges = [[0**2, 1e5**2], [0**2, 25**2], [25**2, 50**2], + [50**2, 100**2], [100**2, 200**2], [200**2, 300**2], + [300**2, 1e5**2]] area_counts = [] for area_name, area_range in zip(area_names[1:], area_ranges[1:]): area_count = 0 @@ -229,24 +248,31 @@ class IMDB(object): boxes = roidb[i]['boxes'][non_gt_inds, :] else: boxes = candidate_boxes[i] - boxes_areas = (boxes[:, 2] - boxes[:, 0] + 1) * (boxes[:, 3] - boxes[:, 1] + 1) - valid_range_inds = np.where((boxes_areas >= area_range[0]) & (boxes_areas < area_range[1]))[0] + boxes_areas = (boxes[:, 2] - boxes[:, 0] + + 1) * (boxes[:, 3] - boxes[:, 1] + 1) + valid_range_inds = np.where((boxes_areas >= area_range[0]) + & (boxes_areas < area_range[1]))[0] area_count += len(valid_range_inds) area_counts.append(area_count) total_counts = float(sum(area_counts)) for area_name, area_count in zip(area_names[1:], area_counts): - logger.info('percentage of %s is %f' % (area_name, area_count / total_counts)) - logger.info('average number of proposal is %f' % (total_counts / self.num_images)) + logger.info('percentage of %s is %f' % + (area_name, area_count / total_counts)) + logger.info('average number of proposal is %f' % + (total_counts / self.num_images)) for area_name, area_range in zip(area_names, area_ranges): gt_overlaps = np.zeros(0) num_pos = 0 for i in range(self.num_images): # check for max_overlaps == 1 avoids including crowd annotations max_gt_overlaps = roidb[i]['gt_overlaps'].max(axis=1) - gt_inds = np.where((roidb[i]['gt_classes'] > 0) & (max_gt_overlaps == 1))[0] + gt_inds = np.where((roidb[i]['gt_classes'] > 0) + & (max_gt_overlaps == 1))[0] gt_boxes = roidb[i]['boxes'][gt_inds, :] - gt_areas = (gt_boxes[:, 2] - gt_boxes[:, 0] + 1) * (gt_boxes[:, 3] - gt_boxes[:, 1] + 1) - valid_gt_inds = np.where((gt_areas >= area_range[0]) & (gt_areas < area_range[1]))[0] + gt_areas = (gt_boxes[:, 2] - gt_boxes[:, 0] + + 1) * (gt_boxes[:, 3] - gt_boxes[:, 1] + 1) + valid_gt_inds = np.where((gt_areas >= area_range[0]) + & (gt_areas < area_range[1]))[0] gt_boxes = gt_boxes[valid_gt_inds, :] num_pos += len(valid_gt_inds) @@ -259,7 +285,8 @@ class IMDB(object): if boxes.shape[0] == 0: continue - overlaps = bbox_overlaps(boxes.astype(np.float), gt_boxes.astype(np.float)) + overlaps = bbox_overlaps(boxes.astype(np.float), + gt_boxes.astype(np.float)) _gt_overlaps = np.zeros((gt_boxes.shape[0])) # choose whatever is smaller to iterate @@ -272,7 +299,8 @@ class IMDB(object): # find which gt box is covered by most IoU gt_ind = max_overlaps.argmax() gt_ovr = max_overlaps.max() - assert (gt_ovr >= 0), '%s\n%s\n%s' % (boxes, gt_boxes, overlaps) + assert (gt_ovr >= + 0), '%s\n%s\n%s' % (boxes, gt_boxes, overlaps) # find the proposal box that covers the best covered gt box box_ind = argmax_overlaps[gt_ind] # record the IoU coverage of this gt box @@ -296,7 +324,8 @@ class IMDB(object): ar = recalls.mean() # print results - print('average recall for {}: {:.3f}, number:{}'.format(area_name, ar, num_pos)) + print('average recall for {}: {:.3f}, number:{}'.format( + area_name, ar, num_pos)) for threshold, recall in zip(thresholds, recalls): print('recall @{:.2f}: {:.3f}'.format(threshold, recall)) @@ -311,8 +340,12 @@ class IMDB(object): assert len(a) == len(b) for i in range(len(a)): a[i]['boxes'] = np.vstack((a[i]['boxes'], b[i]['boxes'])) - a[i]['gt_classes'] = np.hstack((a[i]['gt_classes'], b[i]['gt_classes'])) - a[i]['gt_overlaps'] = np.vstack((a[i]['gt_overlaps'], b[i]['gt_overlaps'])) - a[i]['max_classes'] = np.hstack((a[i]['max_classes'], b[i]['max_classes'])) - a[i]['max_overlaps'] = np.hstack((a[i]['max_overlaps'], b[i]['max_overlaps'])) + a[i]['gt_classes'] = np.hstack( + (a[i]['gt_classes'], b[i]['gt_classes'])) + a[i]['gt_overlaps'] = np.vstack( + (a[i]['gt_overlaps'], b[i]['gt_overlaps'])) + a[i]['max_classes'] = np.hstack( + (a[i]['max_classes'], b[i]['max_classes'])) + a[i]['max_overlaps'] = np.hstack( + (a[i]['max_overlaps'], b[i]['max_overlaps'])) return a diff --git a/detection/RetinaFace/rcnn/dataset/retinaface.py b/detection/RetinaFace/rcnn/dataset/retinaface.py new file mode 100644 index 0000000..6e3b856 --- /dev/null +++ b/detection/RetinaFace/rcnn/dataset/retinaface.py @@ -0,0 +1,197 @@ +from __future__ import print_function +try: + import cPickle as pickle +except ImportError: + import pickle +import cv2 +import os +import numpy as np +import json +#from PIL import Image + +from ..logger import logger +from .imdb import IMDB +from .ds_utils import unique_boxes, filter_small_boxes +from ..config import config + + +class retinaface(IMDB): + def __init__(self, image_set, root_path, data_path): + super(retinaface, self).__init__('retinaface', image_set, root_path, + data_path) + #assert image_set=='train' + + split = image_set + self._split = image_set + self._image_set = image_set + + self.root_path = root_path + self.data_path = data_path + + self._dataset_path = self.data_path + self._imgs_path = os.path.join(self._dataset_path, image_set, 'images') + self._fp_bbox_map = {} + label_file = os.path.join(self._dataset_path, image_set, 'label.txt') + name = None + for line in open(label_file, 'r'): + line = line.strip() + if line.startswith('#'): + name = line[1:].strip() + self._fp_bbox_map[name] = [] + continue + assert name is not None + assert name in self._fp_bbox_map + self._fp_bbox_map[name].append(line) + print('origin image size', len(self._fp_bbox_map)) + + #self.num_images = len(self._image_paths) + #self._image_index = range(len(self._image_paths)) + self.classes = ['bg', 'face'] + self.num_classes = len(self.classes) + + def gt_roidb(self): + cache_file = os.path.join( + self.cache_path, + '{}_{}_gt_roidb.pkl'.format(self.name, self._split)) + if os.path.exists(cache_file): + with open(cache_file, 'rb') as fid: + roidb = pickle.load(fid) + print('{} gt roidb loaded from {}'.format(self.name, cache_file)) + self.num_images = len(roidb) + return roidb + + roidb = [] + max_num_boxes = 0 + nonattr_box_num = 0 + landmark_num = 0 + + pp = 0 + for fp in self._fp_bbox_map: + pp += 1 + if pp % 1000 == 0: + print('loading', pp) + if self._split == 'test': + image_path = os.path.join(self._imgs_path, fp) + roi = {'image': image_path} + roidb.append(roi) + continue + boxes = np.zeros([len(self._fp_bbox_map[fp]), 4], np.float) + landmarks = np.zeros([len(self._fp_bbox_map[fp]), 5, 3], np.float) + blur = np.zeros((len(self._fp_bbox_map[fp]), ), np.float) + boxes_mask = [] + + gt_classes = np.ones([len(self._fp_bbox_map[fp])], np.int32) + overlaps = np.zeros([len(self._fp_bbox_map[fp]), 2], np.float) + + imsize = cv2.imread(os.path.join(self._imgs_path, + fp)).shape[0:2][::-1] + ix = 0 + + for aline in self._fp_bbox_map[fp]: + #imsize = Image.open(os.path.join(self._imgs_path, fp)).size + values = [float(x) for x in aline.strip().split()] + bbox = [ + values[0], values[1], values[0] + values[2], + values[1] + values[3] + ] + + x1 = bbox[0] + y1 = bbox[1] + x2 = min(imsize[0], bbox[2]) + y2 = min(imsize[1], bbox[3]) + if x1 >= x2 or y1 >= y2: + continue + + if config.BBOX_MASK_THRESH > 0: + if ( + x2 - x1 + ) < config.BBOX_MASK_THRESH or y2 - y1 < config.BBOX_MASK_THRESH: + boxes_mask.append(np.array([x1, y1, x2, y2], np.float)) + continue + if ( + x2 - x1 + ) < config.TRAIN.MIN_BOX_SIZE or y2 - y1 < config.TRAIN.MIN_BOX_SIZE: + continue + + boxes[ix, :] = np.array([x1, y1, x2, y2], np.float) + if self._split == 'train': + landmark = np.array(values[4:19], + dtype=np.float32).reshape((5, 3)) + for li in range(5): + #print(landmark) + if landmark[li][0] == -1. and landmark[li][ + 1] == -1.: #missing landmark + assert landmark[li][2] == -1 + else: + assert landmark[li][2] >= 0 + if li == 0: + landmark_num += 1 + if landmark[li][2] == 0.0: #visible + landmark[li][2] = 1.0 + else: + landmark[li][2] = 0.0 + + landmarks[ix] = landmark + + blur[ix] = values[19] + #print(aline, blur[ix]) + if blur[ix] < 0.0: + blur[ix] = 0.3 + nonattr_box_num += 1 + + cls = int(1) + gt_classes[ix] = cls + overlaps[ix, cls] = 1.0 + ix += 1 + max_num_boxes = max(max_num_boxes, ix) + #overlaps = scipy.sparse.csr_matrix(overlaps) + if self._split == 'train' and ix == 0: + continue + boxes = boxes[:ix, :] + landmarks = landmarks[:ix, :, :] + blur = blur[:ix] + gt_classes = gt_classes[:ix] + overlaps = overlaps[:ix, :] + image_path = os.path.join(self._imgs_path, fp) + with open(image_path, 'rb') as fin: + stream = fin.read() + stream = np.fromstring(stream, dtype=np.uint8) + + roi = { + 'image': image_path, + 'stream': stream, + 'height': imsize[1], + 'width': imsize[0], + 'boxes': boxes, + 'landmarks': landmarks, + 'blur': blur, + 'gt_classes': gt_classes, + 'gt_overlaps': overlaps, + 'max_classes': overlaps.argmax(axis=1), + 'max_overlaps': overlaps.max(axis=1), + 'flipped': False, + } + if len(boxes_mask) > 0: + boxes_mask = np.array(boxes_mask) + roi['boxes_mask'] = boxes_mask + roidb.append(roi) + for roi in roidb: + roi['max_num_boxes'] = max_num_boxes + self.num_images = len(roidb) + print('roidb size', len(roidb)) + print('non attr box num', nonattr_box_num) + print('landmark num', landmark_num) + with open(cache_file, 'wb') as fid: + pickle.dump(roidb, fid, pickle.HIGHEST_PROTOCOL) + print('wrote gt roidb to {}'.format(cache_file)) + + return roidb + + def write_detections(self, all_boxes, output_dir='./output/'): + pass + + def evaluate_detections(self, + all_boxes, + output_dir='./output/', + method_name='insightdetection'): + pass diff --git a/RetinaFace/rcnn/io/__init__.py b/detection/RetinaFace/rcnn/io/__init__.py similarity index 100% rename from RetinaFace/rcnn/io/__init__.py rename to detection/RetinaFace/rcnn/io/__init__.py diff --git a/detection/RetinaFace/rcnn/io/image.py b/detection/RetinaFace/rcnn/io/image.py new file mode 100644 index 0000000..0296fb4 --- /dev/null +++ b/detection/RetinaFace/rcnn/io/image.py @@ -0,0 +1,886 @@ +from __future__ import print_function +import numpy as np +import cv2 +import os +import math +import sys +import random +from ..config import config + + +def brightness_aug(src, x): + alpha = 1.0 + random.uniform(-x, x) + src *= alpha + return src + + +def contrast_aug(src, x): + alpha = 1.0 + random.uniform(-x, x) + coef = np.array([[[0.299, 0.587, 0.114]]]) + gray = src * coef + gray = (3.0 * (1.0 - alpha) / gray.size) * np.sum(gray) + src *= alpha + src += gray + return src + + +def saturation_aug(src, x): + alpha = 1.0 + random.uniform(-x, x) + coef = np.array([[[0.299, 0.587, 0.114]]]) + gray = src * coef + gray = np.sum(gray, axis=2, keepdims=True) + gray *= (1.0 - alpha) + src *= alpha + src += gray + return src + + +def color_aug(img, x): + if config.COLOR_MODE > 1: + augs = [brightness_aug, contrast_aug, saturation_aug] + random.shuffle(augs) + else: + augs = [brightness_aug] + for aug in augs: + #print(img.shape) + img = aug(img, x) + #print(img.shape) + return img + + +def get_image(roidb, scale=False): + """ + preprocess image and return processed roidb + :param roidb: a list of roidb + :return: list of img as in mxnet format + roidb add new item['im_info'] + 0 --- x (width, second dim of im) + | + y (height, first dim of im) + """ + num_images = len(roidb) + processed_ims = [] + processed_roidb = [] + for i in range(num_images): + roi_rec = roidb[i] + if 'stream' in roi_rec: + im = cv2.imdecode(roi_rec['stream'], cv2.IMREAD_COLOR) + else: + assert os.path.exists( + roi_rec['image']), '{} does not exist'.format(roi_rec['image']) + im = cv2.imread(roi_rec['image']) + if roidb[i]['flipped']: + im = im[:, ::-1, :] + new_rec = roi_rec.copy() + if scale: + scale_range = config.TRAIN.SCALE_RANGE + im_scale = np.random.uniform(scale_range[0], scale_range[1]) + im = cv2.resize(im, + None, + None, + fx=im_scale, + fy=im_scale, + interpolation=cv2.INTER_LINEAR) + elif not config.ORIGIN_SCALE: + scale_ind = random.randrange(len(config.SCALES)) + target_size = config.SCALES[scale_ind][0] + max_size = config.SCALES[scale_ind][1] + im, im_scale = resize(im, + target_size, + max_size, + stride=config.IMAGE_STRIDE) + else: + im_scale = 1.0 + im_tensor = transform(im, config.PIXEL_MEANS, config.PIXEL_STDS) + if 'boxes_mask' in roi_rec: + im = im.astype(np.float32) + boxes_mask = roi_rec['boxes_mask'].copy() * im_scale + boxes_mask = boxes_mask.astype(np.int) + for j in range(boxes_mask.shape[0]): + m = boxes_mask[j] + im_tensor[:, :, m[1]:m[3], m[0]:m[2]] = 0.0 + #print('find mask', m, file=sys.stderr) + processed_ims.append(im_tensor) + new_rec['boxes'] = roi_rec['boxes'].copy() * im_scale + if config.TRAIN.IMAGE_ALIGN > 0: + if im_tensor.shape[ + 2] % config.TRAIN.IMAGE_ALIGN != 0 or im_tensor.shape[ + 3] % config.TRAIN.IMAGE_ALIGN != 0: + new_height = math.ceil( + float(im_tensor.shape[2]) / + config.TRAIN.IMAGE_ALIGN) * config.TRAIN.IMAGE_ALIGN + new_width = math.ceil( + float(im_tensor.shape[3]) / + config.TRAIN.IMAGE_ALIGN) * config.TRAIN.IMAGE_ALIGN + new_im_tensor = np.zeros( + (1, 3, int(new_height), int(new_width))) + new_im_tensor[:, :, 0:im_tensor.shape[2], + 0:im_tensor.shape[3]] = im_tensor + print(im_tensor.shape, new_im_tensor.shape, file=sys.stderr) + im_tensor = new_im_tensor + #print('boxes', new_rec['boxes'], file=sys.stderr) + im_info = [im_tensor.shape[2], im_tensor.shape[3], im_scale] + new_rec['im_info'] = im_info + processed_roidb.append(new_rec) + return processed_ims, processed_roidb + + +TMP_ID = -1 + + +#bakup method +def __get_crop_image(roidb): + """ + preprocess image and return processed roidb + :param roidb: a list of roidb + :return: list of img as in mxnet format + roidb add new item['im_info'] + 0 --- x (width, second dim of im) + | + y (height, first dim of im) + """ + #roidb and each roi_rec can not be changed as it will be reused in next epoch + num_images = len(roidb) + processed_ims = [] + processed_roidb = [] + for i in range(num_images): + roi_rec = roidb[i] + if 'stream' in roi_rec: + im = cv2.imdecode(roi_rec['stream'], cv2.IMREAD_COLOR) + else: + assert os.path.exists( + roi_rec['image']), '{} does not exist'.format(roi_rec['image']) + im = cv2.imread(roi_rec['image']) + if roidb[i]['flipped']: + im = im[:, ::-1, :] + if 'boxes_mask' in roi_rec: + #im = im.astype(np.float32) + boxes_mask = roi_rec['boxes_mask'].copy() + boxes_mask = boxes_mask.astype(np.int) + for j in range(boxes_mask.shape[0]): + m = boxes_mask[j] + im[m[1]:m[3], m[0]:m[2], :] = 0 + #print('find mask', m, file=sys.stderr) + new_rec = roi_rec.copy() + + #choose one gt randomly + SIZE = config.SCALES[0][0] + TARGET_BOX_SCALES = np.array([16, 32, 64, 128, 256, 512]) + assert roi_rec['boxes'].shape[0] > 0 + candidates = [] + for i in range(roi_rec['boxes'].shape[0]): + box = roi_rec['boxes'][i] + box_size = max(box[2] - box[0], box[3] - box[1]) + if box_size < config.TRAIN.MIN_BOX_SIZE: + continue + #if box[0]<0 or box[1]<0: + # continue + #if box[2]>im.shape[1] or box[3]>im.shape[0]: + # continue; + candidates.append(i) + assert len(candidates) > 0 + box_ind = random.choice(candidates) + box = roi_rec['boxes'][box_ind] + box_size = max(box[2] - box[0], box[3] - box[1]) + dist = np.abs(TARGET_BOX_SCALES - box_size) + nearest = np.argmin(dist) + target_ind = random.randrange(min(len(TARGET_BOX_SCALES), nearest + 2)) + target_box_size = TARGET_BOX_SCALES[target_ind] + im_scale = float(target_box_size) / box_size + #min_scale = float(SIZE)/np.min(im.shape[0:2]) + #if im_scale= im.shape[ + 1] or center[1] >= im.shape[0]: + continue + if box_size < config.TRAIN.MIN_BOX_SIZE: + continue + boxes_new.append(box) + classes_new.append(new_rec['gt_classes'][i]) + new_rec['boxes'] = np.array(boxes_new) + new_rec['gt_classes'] = np.array(classes_new) + #print('after', new_rec['boxes'].shape[0]) + #assert new_rec['boxes'].shape[0]>0 + DEBUG = True + if DEBUG: + global TMP_ID + if TMP_ID < 10: + tim = im.copy() + for i in range(new_rec['boxes'].shape[0]): + box = new_rec['boxes'][i].copy().astype(np.int) + cv2.rectangle(tim, (box[0], box[1]), (box[2], box[3]), + (255, 0, 0), 1) + filename = './trainimages/train%d.png' % TMP_ID + TMP_ID += 1 + cv2.imwrite(filename, tim) + + im_tensor = transform(im, config.PIXEL_MEANS, config.PIXEL_STDS, + config.PIXEL_SCALE) + + processed_ims.append(im_tensor) + #print('boxes', new_rec['boxes'], file=sys.stderr) + im_info = [im_tensor.shape[2], im_tensor.shape[3], im_scale] + new_rec['im_info'] = im_info + processed_roidb.append(new_rec) + return processed_ims, processed_roidb + + +def expand_bboxes(bboxes, + image_width, + image_height, + expand_left=2., + expand_up=2., + expand_right=2., + expand_down=2.): + """ + Expand bboxes, expand 2 times by defalut. + """ + expand_boxes = [] + for bbox in bboxes: + xmin = bbox[0] + ymin = bbox[1] + xmax = bbox[2] + ymax = bbox[3] + w = xmax - xmin + h = ymax - ymin + ex_xmin = max(xmin - w / expand_left, 0.) + ex_ymin = max(ymin - h / expand_up, 0.) + ex_xmax = min(xmax + w / expand_right, image_width) + ex_ymax = min(ymax + h / expand_down, image_height) + expand_boxes.append([ex_xmin, ex_ymin, ex_xmax, ex_ymax]) + return expand_boxes + + +def get_crop_image1(roidb): + """ + preprocess image and return processed roidb + :param roidb: a list of roidb + :return: list of img as in mxnet format + roidb add new item['im_info'] + 0 --- x (width, second dim of im) + | + y (height, first dim of im) + """ + #roidb and each roi_rec can not be changed as it will be reused in next epoch + num_images = len(roidb) + processed_ims = [] + processed_roidb = [] + for i in range(num_images): + roi_rec = roidb[i] + if 'stream' in roi_rec: + im = cv2.imdecode(roi_rec['stream'], cv2.IMREAD_COLOR) + else: + assert os.path.exists( + roi_rec['image']), '{} does not exist'.format(roi_rec['image']) + im = cv2.imread(roi_rec['image']) + if roidb[i]['flipped']: + im = im[:, ::-1, :] + if 'boxes_mask' in roi_rec: + #im = im.astype(np.float32) + boxes_mask = roi_rec['boxes_mask'].copy() + boxes_mask = boxes_mask.astype(np.int) + for j in range(boxes_mask.shape[0]): + m = boxes_mask[j] + im[m[1]:m[3], m[0]:m[2], :] = 127 + #print('find mask', m, file=sys.stderr) + SIZE = config.SCALES[0][0] + PRE_SCALES = [0.3, 0.45, 0.6, 0.8, 1.0] + #PRE_SCALES = [0.3, 0.45, 0.6, 0.8, 1.0, 0.8, 1.0, 0.8, 1.0] + _scale = random.choice(PRE_SCALES) + #_scale = np.random.uniform(PRE_SCALES[0], PRE_SCALES[-1]) + size = int(np.min(im.shape[0:2]) * _scale) + #size = int(np.round(_scale*np.min(im.shape[0:2]))) + im_scale = float(SIZE) / size + #origin_im_scale = im_scale + #size = np.round(np.min(im.shape[0:2])*im_scale) + #im_scale *= (float(SIZE)/size) + origin_shape = im.shape + if _scale > 10.0: #avoid im.size= SIZE and im.shape[1] >= SIZE + #print('image size', origin_shape, _scale, SIZE, size, im_scale) + + new_rec = roi_rec.copy() + new_rec['boxes'] = roi_rec['boxes'].copy() * im_scale + if config.FACE_LANDMARK: + new_rec['landmarks'] = roi_rec['landmarks'].copy() + new_rec['landmarks'][:, :, 0:2] *= im_scale + retry = 0 + LIMIT = 25 + size = SIZE + while retry < LIMIT: + up, left = (np.random.randint(0, im.shape[0] - size + 1), + np.random.randint(0, im.shape[1] - size + 1)) + boxes_new = new_rec['boxes'].copy() + im_new = im[up:(up + size), left:(left + size), :] + #print('crop', up, left, size, im_scale) + boxes_new[:, 0] -= left + boxes_new[:, 2] -= left + boxes_new[:, 1] -= up + boxes_new[:, 3] -= up + if config.FACE_LANDMARK: + landmarks_new = new_rec['landmarks'].copy() + landmarks_new[:, :, 0] -= left + landmarks_new[:, :, 1] -= up + #for i in range(0,10,2): + # landmarks_new[:,i] -= left + #for i in range(1,10,2): + # landmarks_new[:,i] -= up + valid_landmarks = [] + #im_new = cv2.resize(im_new, (SIZE, SIZE), interpolation=cv2.INTER_LINEAR) + #boxes_new *= im_scale + #print(origin_shape, im_new.shape, im_scale) + valid = [] + valid_boxes = [] + for i in range(boxes_new.shape[0]): + box = boxes_new[i] + #center = np.array(([box[0], box[1]]+[box[2], box[3]]))/2 + centerx = (box[0] + box[2]) / 2 + centery = (box[1] + box[3]) / 2 + + #box[0] = max(0, box[0]) + #box[1] = max(0, box[1]) + #box[2] = min(im_new.shape[1], box[2]) + #box[3] = min(im_new.shape[0], box[3]) + box_size = max(box[2] - box[0], box[3] - box[1]) + + if centerx < 0 or centery < 0 or centerx >= im_new.shape[ + 1] or centery >= im_new.shape[0]: + continue + if box_size < config.TRAIN.MIN_BOX_SIZE: + continue + #filter by landmarks? TODO + valid.append(i) + valid_boxes.append(box) + if config.FACE_LANDMARK: + valid_landmarks.append(landmarks_new[i]) + if len(valid) > 0 or retry == LIMIT - 1: + im = im_new + new_rec['boxes'] = np.array(valid_boxes) + new_rec['gt_classes'] = new_rec['gt_classes'][valid] + if config.FACE_LANDMARK: + new_rec['landmarks'] = np.array(valid_landmarks) + if config.HEAD_BOX: + face_box = new_rec['boxes'] + head_box = expand_bboxes(face_box, + image_width=im.shape[1], + image_height=im.shape[0]) + new_rec['boxes_head'] = np.array(head_box) + break + + retry += 1 + + if config.COLOR_MODE > 0 and config.COLOR_JITTERING > 0.0: + im = im.astype(np.float32) + im = color_aug(im, config.COLOR_JITTERING) + + #assert np.all(new_rec['landmarks'][:,10]>0.0) + global TMP_ID + if TMP_ID >= 0 and TMP_ID < 10: + tim = im.copy().astype(np.uint8) + for i in range(new_rec['boxes'].shape[0]): + box = new_rec['boxes'][i].copy().astype(np.int) + cv2.rectangle(tim, (box[0], box[1]), (box[2], box[3]), + (255, 0, 0), 1) + print('draw box:', box) + if config.FACE_LANDMARK: + for i in range(new_rec['landmarks'].shape[0]): + landmark = new_rec['landmarks'][i].copy() + if landmark[0][2] < 0: + print('zero', landmark) + continue + landmark = landmark.astype(np.int) + print('draw landmark', landmark) + for k in range(5): + color = (0, 0, 255) + if k == 0 or k == 3: + color = (0, 255, 0) + pp = (landmark[k][0], landmark[k][1]) + cv2.circle(tim, (pp[0], pp[1]), 1, color, 2) + filename = './trainimages/train%d.png' % TMP_ID + print('write', filename) + cv2.imwrite(filename, tim) + TMP_ID += 1 + + im_tensor = transform(im, config.PIXEL_MEANS, config.PIXEL_STDS, + config.PIXEL_SCALE) + + processed_ims.append(im_tensor) + #print('boxes', new_rec['boxes'], file=sys.stderr) + im_info = [im_tensor.shape[2], im_tensor.shape[3], im_scale] + new_rec['im_info'] = np.array(im_info, dtype=np.float32) + processed_roidb.append(new_rec) + return processed_ims, processed_roidb + + +def get_crop_image2(roidb): + """ + preprocess image and return processed roidb + :param roidb: a list of roidb + :return: list of img as in mxnet format + roidb add new item['im_info'] + 0 --- x (width, second dim of im) + | + y (height, first dim of im) + """ + #roidb and each roi_rec can not be changed as it will be reused in next epoch + num_images = len(roidb) + processed_ims = [] + processed_roidb = [] + for i in range(num_images): + roi_rec = roidb[i] + if 'stream' in roi_rec: + im = cv2.imdecode(roi_rec['stream'], cv2.IMREAD_COLOR) + else: + assert os.path.exists( + roi_rec['image']), '{} does not exist'.format(roi_rec['image']) + im = cv2.imread(roi_rec['image']) + if roidb[i]['flipped']: + im = im[:, ::-1, :] + if 'boxes_mask' in roi_rec: + #im = im.astype(np.float32) + boxes_mask = roi_rec['boxes_mask'].copy() + boxes_mask = boxes_mask.astype(np.int) + for j in range(boxes_mask.shape[0]): + m = boxes_mask[j] + im[m[1]:m[3], m[0]:m[2], :] = 0 + #print('find mask', m, file=sys.stderr) + SIZE = config.SCALES[0][0] + scale_array = np.array([16, 32, 64, 128, 256, 512], dtype=np.float32) + candidates = [] + for i in range(roi_rec['boxes'].shape[0]): + box = roi_rec['boxes'][i] + box_size = max(box[2] - box[0], box[3] - box[1]) + if box_size < config.TRAIN.MIN_BOX_SIZE: + continue + #if box[0]<0 or box[1]<0: + # continue + #if box[2]>im.shape[1] or box[3]>im.shape[0]: + # continue; + candidates.append(i) + assert len(candidates) > 0 + box_ind = random.choice(candidates) + box = roi_rec['boxes'][box_ind] + width = box[2] - box[0] + height = box[3] - box[1] + wid = width + hei = height + resize_width, resize_height = config.SCALES[0] + image_width = im.shape[0] + image_height = im.shape[1] + area = width * height + range_size = 0 + for scale_ind in range(0, len(scale_array) - 1): + if area > scale_array[scale_ind] ** 2 and area < \ + scale_array[scale_ind + 1] ** 2: + range_size = scale_ind + 1 + break + + if area > scale_array[len(scale_array) - 2]**2: + range_size = len(scale_array) - 2 + scale_choose = 0.0 + if range_size == 0: + rand_idx_size = 0 + else: + # np.random.randint range: [low, high) + rng_rand_size = np.random.randint(0, range_size + 1) + rand_idx_size = rng_rand_size % (range_size + 1) + + if rand_idx_size == range_size: + min_resize_val = scale_array[rand_idx_size] / 2.0 + max_resize_val = min(2.0 * scale_array[rand_idx_size], + 2 * math.sqrt(wid * hei)) + scale_choose = random.uniform(min_resize_val, max_resize_val) + else: + min_resize_val = scale_array[rand_idx_size] / 2.0 + max_resize_val = 2.0 * scale_array[rand_idx_size] + scale_choose = random.uniform(min_resize_val, max_resize_val) + + sample_bbox_size = wid * resize_width / scale_choose + + w_off_orig = 0.0 + h_off_orig = 0.0 + if sample_bbox_size < max(image_height, image_width): + if wid <= sample_bbox_size: + w_off_orig = np.random.uniform(xmin + wid - sample_bbox_size, + xmin) + else: + w_off_orig = np.random.uniform(xmin, + xmin + wid - sample_bbox_size) + + if hei <= sample_bbox_size: + h_off_orig = np.random.uniform(ymin + hei - sample_bbox_size, + ymin) + else: + h_off_orig = np.random.uniform(ymin, + ymin + hei - sample_bbox_size) + + else: + w_off_orig = np.random.uniform(image_width - sample_bbox_size, 0.0) + h_off_orig = np.random.uniform(image_height - sample_bbox_size, + 0.0) + + w_off_orig = math.floor(w_off_orig) + h_off_orig = math.floor(h_off_orig) + + # Figure out top left coordinates. + w_off = 0.0 + h_off = 0.0 + w_off = float(w_off_orig / image_width) + h_off = float(h_off_orig / image_height) + im_new = im[up:(up + size), left:(left + size), :] + + sampled_bbox = bbox(w_off, h_off, + w_off + float(sample_bbox_size / image_width), + h_off + float(sample_bbox_size / image_height)) + return sampled_bbox + + box_size = max(box[2] - box[0], box[3] - box[1]) + dist = np.abs(TARGET_BOX_SCALES - box_size) + nearest = np.argmin(dist) + target_ind = random.randrange(min(len(TARGET_BOX_SCALES), nearest + 2)) + target_box_size = TARGET_BOX_SCALES[target_ind] + im_scale = float(target_box_size) / box_size + PRE_SCALES = [0.3, 0.45, 0.6, 0.8, 1.0] + _scale = random.choice(PRE_SCALES) + #_scale = np.random.uniform(PRE_SCALES[0], PRE_SCALES[-1]) + size = int(np.round(_scale * np.min(im.shape[0:2]))) + im_scale = float(SIZE) / size + #origin_im_scale = im_scale + #size = np.round(np.min(im.shape[0:2])*im_scale) + #im_scale *= (float(SIZE)/size) + origin_shape = im.shape + if _scale > 10.0: #avoid im.size= SIZE and im.shape[1] >= SIZE + + new_rec = roi_rec.copy() + new_rec['boxes'] = roi_rec['boxes'].copy() * im_scale + if config.FACE_LANDMARK: + new_rec['landmarks'] = roi_rec['landmarks'].copy() * im_scale + retry = 0 + LIMIT = 25 + size = SIZE + while retry < LIMIT: + up, left = (np.random.randint(0, im.shape[0] - size + 1), + np.random.randint(0, im.shape[1] - size + 1)) + boxes_new = new_rec['boxes'].copy() + im_new = im[up:(up + size), left:(left + size), :] + #print('crop', up, left, size, im_scale) + boxes_new[:, 0] -= left + boxes_new[:, 2] -= left + boxes_new[:, 1] -= up + boxes_new[:, 3] -= up + if config.FACE_LANDMARK: + landmarks_new = new_rec['landmarks'].copy() + for i in range(0, 10, 2): + landmarks_new[:, i] -= left + for i in range(1, 10, 2): + landmarks_new[:, i] -= up + valid_landmarks = [] + #im_new = cv2.resize(im_new, (SIZE, SIZE), interpolation=cv2.INTER_LINEAR) + #boxes_new *= im_scale + #print(origin_shape, im_new.shape, im_scale) + valid = [] + valid_boxes = [] + for i in range(boxes_new.shape[0]): + box = boxes_new[i] + #center = np.array(([box[0], box[1]]+[box[2], box[3]]))/2 + centerx = (box[0] + box[2]) / 2 + centery = (box[1] + box[3]) / 2 + + #box[0] = max(0, box[0]) + #box[1] = max(0, box[1]) + #box[2] = min(im_new.shape[1], box[2]) + #box[3] = min(im_new.shape[0], box[3]) + box_size = max(box[2] - box[0], box[3] - box[1]) + + if centerx < 0 or centery < 0 or centerx >= im_new.shape[ + 1] or centery >= im_new.shape[0]: + continue + if box_size < config.TRAIN.MIN_BOX_SIZE: + continue + #filter by landmarks? TODO + valid.append(i) + valid_boxes.append(box) + if config.FACE_LANDMARK: + valid_landmarks.append(landmarks_new[i]) + if len(valid) > 0 or retry == LIMIT - 1: + im = im_new + new_rec['boxes'] = np.array(valid_boxes) + new_rec['gt_classes'] = new_rec['gt_classes'][valid] + if config.FACE_LANDMARK: + new_rec['landmarks'] = np.array(valid_landmarks) + break + + retry += 1 + + if config.COLOR_JITTERING > 0.0: + im = im.astype(np.float32) + im = color_aug(im, config.COLOR_JITTERING) + + #assert np.all(new_rec['landmarks'][:,10]>0.0) + global TMP_ID + if TMP_ID >= 0 and TMP_ID < 10: + tim = im.copy().astype(np.uint8) + for i in range(new_rec['boxes'].shape[0]): + box = new_rec['boxes'][i].copy().astype(np.int) + cv2.rectangle(tim, (box[0], box[1]), (box[2], box[3]), + (255, 0, 0), 1) + print('draw box:', box) + if config.FACE_LANDMARK: + for i in range(new_rec['landmarks'].shape[0]): + landmark = new_rec['landmarks'][i].copy() + if landmark[10] == 0.0: + print('zero', landmark) + continue + landmark = landmark.astype(np.int) + print('draw landmark', landmark) + for k in range(5): + color = (0, 0, 255) + if k == 0 or k == 3: + color = (0, 255, 0) + pp = (landmark[k * 2], landmark[1 + k * 2]) + cv2.circle(tim, (pp[0], pp[1]), 1, color, 2) + filename = './trainimages/train%d.png' % TMP_ID + print('write', filename) + cv2.imwrite(filename, tim) + TMP_ID += 1 + + im_tensor = transform(im, config.PIXEL_MEANS, config.PIXEL_STDS, + config.PIXEL_SCALE) + + processed_ims.append(im_tensor) + #print('boxes', new_rec['boxes'], file=sys.stderr) + im_info = [im_tensor.shape[2], im_tensor.shape[3], im_scale] + new_rec['im_info'] = np.array(im_info, dtype=np.float32) + processed_roidb.append(new_rec) + return processed_ims, processed_roidb + + +def do_mixup(im1, roidb1, im2, roidb2): + im = (im1 + im2) / 2.0 + roidb = {} + #print(roidb1.keys()) + #for k in roidb1: + for k in ['boxes', 'landmarks', 'gt_classes', 'im_info']: + v1 = roidb1[k] + v2 = roidb2[k] + if k != 'im_info': + #print('try', k, v1.shape, v2.shape) + if v1.shape[0] > 0 and v2.shape[0] > 0: + v = np.concatenate((v1, v2), axis=0) + else: + v = v1 + else: + v = v1 + #print(k, v1.shape, v2.shape, v.shape) + roidb[k] = v + return im, roidb + + +def get_crop_image(roidb): + ims, roidbs = get_crop_image1(roidb) + if config.MIXUP > 0.0 and np.random.random() < config.MIXUP: + for i in range(len(ims)): + im = ims[i] + roidb = roidbs[i] + j = np.random.randint(0, len(ims) - 1) + if j >= i: + j += 1 + im, roidb = do_mixup(im, roidb, ims[j], roidbs[j]) + ims[i] = im + roidbs[i] = roidb + return ims, roidbs + + +def resize(im, target_size, max_size, stride=0, min_size=0): + """ + only resize input image to target size and return scale + :param im: BGR image input by opencv + :param target_size: one dimensional size (the short side) + :param max_size: one dimensional max size (the long side) + :param stride: if given, pad the image to designated stride + :return: + """ + im_shape = im.shape + im_size_min = np.min(im_shape[0:2]) + im_size_max = np.max(im_shape[0:2]) + im_scale = float(target_size) / float(im_size_min) + # prevent bigger axis from being more than max_size: + if np.round(im_scale * im_size_max) > max_size: + im_scale = float(max_size) / float(im_size_max) + if min_size > 0 and np.round(im_scale * im_size_min) < min_size: + im_scale = float(min_size) / float(im_size_min) + im = cv2.resize(im, + None, + None, + fx=im_scale, + fy=im_scale, + interpolation=cv2.INTER_LINEAR) + + if stride == 0: + return im, im_scale + else: + # pad to product of stride + im_height = int(np.ceil(im.shape[0] / float(stride)) * stride) + im_width = int(np.ceil(im.shape[1] / float(stride)) * stride) + im_channel = im.shape[2] + padded_im = np.zeros((im_height, im_width, im_channel)) + padded_im[:im.shape[0], :im.shape[1], :] = im + return padded_im, im_scale + + +def transform(im, pixel_means, pixel_stds, pixel_scale): + """ + transform into mxnet tensor, + subtract pixel size and transform to correct format + :param im: [height, width, channel] in BGR + :param pixel_means: [B, G, R pixel means] + :return: [batch, channel, height, width] + """ + im_tensor = np.zeros((1, 3, im.shape[0], im.shape[1])) + for i in range(3): + im_tensor[0, i, :, :] = (im[:, :, 2 - i] / pixel_scale - + pixel_means[2 - i]) / pixel_stds[2 - i] + return im_tensor + + +def transform_inverse(im_tensor, pixel_means): + """ + transform from mxnet im_tensor to ordinary RGB image + im_tensor is limited to one image + :param im_tensor: [batch, channel, height, width] + :param pixel_means: [B, G, R pixel means] + :return: im [height, width, channel(RGB)] + """ + assert im_tensor.shape[0] == 1 + im_tensor = im_tensor.copy() + # put channel back + channel_swap = (0, 2, 3, 1) + im_tensor = im_tensor.transpose(channel_swap) + im = im_tensor[0] + assert im.shape[2] == 3 + im += pixel_means[[2, 1, 0]] + im = im.astype(np.uint8) + return im + + +def tensor_vstack(tensor_list, pad=0): + """ + vertically stack tensors + :param tensor_list: list of tensor to be stacked vertically + :param pad: label to pad with + :return: tensor with max shape + """ + ndim = len(tensor_list[0].shape) + dtype = tensor_list[0].dtype + islice = tensor_list[0].shape[0] + dimensions = [] + first_dim = sum([tensor.shape[0] for tensor in tensor_list]) + dimensions.append(first_dim) + for dim in range(1, ndim): + dimensions.append(max([tensor.shape[dim] for tensor in tensor_list])) + if pad == 0: + all_tensor = np.zeros(tuple(dimensions), dtype=dtype) + elif pad == 1: + all_tensor = np.ones(tuple(dimensions), dtype=dtype) + else: + all_tensor = np.full(tuple(dimensions), pad, dtype=dtype) + if ndim == 1: + for ind, tensor in enumerate(tensor_list): + all_tensor[ind * islice:(ind + 1) * islice] = tensor + elif ndim == 2: + for ind, tensor in enumerate(tensor_list): + all_tensor[ind * islice:(ind + 1) * + islice, :tensor.shape[1]] = tensor + elif ndim == 3: + for ind, tensor in enumerate(tensor_list): + all_tensor[ind * islice:(ind + 1) * + islice, :tensor.shape[1], :tensor.shape[2]] = tensor + elif ndim == 4: + for ind, tensor in enumerate(tensor_list): + all_tensor[ind * islice:(ind + 1) * islice, :tensor. + shape[1], :tensor.shape[2], :tensor.shape[3]] = tensor + elif ndim == 5: + for ind, tensor in enumerate(tensor_list): + all_tensor[ind * islice:(ind + 1) * + islice, :tensor.shape[1], :tensor.shape[2], :tensor. + shape[3], :tensor.shape[4]] = tensor + else: + print(tensor_list[0].shape) + raise Exception('Sorry, unimplemented.') + return all_tensor diff --git a/RetinaFace/rcnn/io/rcnn.py b/detection/RetinaFace/rcnn/io/rcnn.py similarity index 74% rename from RetinaFace/rcnn/io/rcnn.py rename to detection/RetinaFace/rcnn/io/rcnn.py index 298df8e..1b3a443 100644 --- a/RetinaFace/rcnn/io/rcnn.py +++ b/detection/RetinaFace/rcnn/io/rcnn.py @@ -37,9 +37,7 @@ def get_rcnn_testbatch(roidb): batch_index = 0 * np.ones((rois.shape[0], 1)) rois_array = np.hstack((batch_index, rois))[np.newaxis, :] - data = {'data': im_array, - 'rois': rois_array, - 'im_info': im_info} + data = {'data': im_array, 'rois': rois_array, 'im_info': im_info} label = {} return data, label @@ -99,17 +97,24 @@ def get_rcnn_batch(roidb): bbox_targets_array = np.array(bbox_targets_array) bbox_weights_array = np.array(bbox_weights_array) - data = {'data': im_array, - 'rois': rois_array} - label = {'label': labels_array, - 'bbox_target': bbox_targets_array, - 'bbox_weight': bbox_weights_array} + data = {'data': im_array, 'rois': rois_array} + label = { + 'label': labels_array, + 'bbox_target': bbox_targets_array, + 'bbox_weight': bbox_weights_array + } return data, label -def sample_rois(rois, fg_rois_per_image, rois_per_image, num_classes, - labels=None, overlaps=None, bbox_targets=None, gt_boxes=None): +def sample_rois(rois, + fg_rois_per_image, + rois_per_image, + num_classes, + labels=None, + overlaps=None, + bbox_targets=None, + gt_boxes=None): """ generate random sample of ROIs comprising foreground and background examples :param rois: all_rois [n, 4]; e2e: [n, 5] with batch_index @@ -123,7 +128,8 @@ def sample_rois(rois, fg_rois_per_image, rois_per_image, num_classes, :return: (labels, rois, bbox_targets, bbox_weights) """ if labels is None: - overlaps = bbox_overlaps(rois[:, 1:].astype(np.float), gt_boxes[:, :4].astype(np.float)) + overlaps = bbox_overlaps(rois[:, 1:].astype(np.float), + gt_boxes[:, :4].astype(np.float)) gt_assignment = overlaps.argmax(axis=1) overlaps = overlaps.max(axis=1) labels = gt_boxes[gt_assignment, 4] @@ -134,16 +140,22 @@ def sample_rois(rois, fg_rois_per_image, rois_per_image, num_classes, fg_rois_per_this_image = np.minimum(fg_rois_per_image, fg_indexes.size) # Sample foreground regions without replacement if len(fg_indexes) > fg_rois_per_this_image: - fg_indexes = npr.choice(fg_indexes, size=fg_rois_per_this_image, replace=False) + fg_indexes = npr.choice(fg_indexes, + size=fg_rois_per_this_image, + replace=False) # Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI) - bg_indexes = np.where((overlaps < config.TRAIN.BG_THRESH_HI) & (overlaps >= config.TRAIN.BG_THRESH_LO))[0] + bg_indexes = np.where((overlaps < config.TRAIN.BG_THRESH_HI) + & (overlaps >= config.TRAIN.BG_THRESH_LO))[0] # Compute number of background RoIs to take from this image (guarding against there being fewer than desired) bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image - bg_rois_per_this_image = np.minimum(bg_rois_per_this_image, bg_indexes.size) + bg_rois_per_this_image = np.minimum(bg_rois_per_this_image, + bg_indexes.size) # Sample foreground regions without replacement if len(bg_indexes) > bg_rois_per_this_image: - bg_indexes = npr.choice(bg_indexes, size=bg_rois_per_this_image, replace=False) + bg_indexes = npr.choice(bg_indexes, + size=bg_rois_per_this_image, + replace=False) # indexes selected keep_indexes = np.append(fg_indexes, bg_indexes) @@ -165,10 +177,11 @@ def sample_rois(rois, fg_rois_per_image, rois_per_image, num_classes, if bbox_targets is not None: bbox_target_data = bbox_targets[keep_indexes, :] else: - targets = bbox_transform(rois[:, 1:], gt_boxes[gt_assignment[keep_indexes], :4]) + targets = bbox_transform(rois[:, 1:], + gt_boxes[gt_assignment[keep_indexes], :4]) if config.TRAIN.BBOX_NORMALIZATION_PRECOMPUTED: - targets = ((targets - np.array(config.TRAIN.BBOX_MEANS)) - / np.array(config.TRAIN.BBOX_STDS)) + targets = ((targets - np.array(config.TRAIN.BBOX_MEANS)) / + np.array(config.TRAIN.BBOX_STDS)) bbox_target_data = np.hstack((labels[:, np.newaxis], targets)) bbox_targets, bbox_weights = \ @@ -176,6 +189,7 @@ def sample_rois(rois, fg_rois_per_image, rois_per_image, num_classes, return rois, labels, bbox_targets, bbox_weights + def get_fpn_rcnn_testbatch(roidb): """ return a dict of testbatch @@ -194,19 +208,22 @@ def get_fpn_rcnn_testbatch(roidb): rois_area = np.sqrt((rois[:, 2] - rois[:, 0]) * (rois[:, 3] - rois[:, 1])) area_threshold = {'P5': 448, 'P4': 224, 'P3': 112} rois_p5 = rois[area_threshold['P5'] <= rois_area] - rois_p4 = rois[np.logical_and(area_threshold['P4'] <= rois_area, rois_area < area_threshold['P5'])] - rois_p3 = rois[np.logical_and(area_threshold['P3'] <= rois_area, rois_area < area_threshold['P4'])] - rois_p2 = rois[np.logical_and(0 < rois_area, rois_area < area_threshold['P3'])] + rois_p4 = rois[np.logical_and(area_threshold['P4'] <= rois_area, + rois_area < area_threshold['P5'])] + rois_p3 = rois[np.logical_and(area_threshold['P3'] <= rois_area, + rois_area < area_threshold['P4'])] + rois_p2 = rois[np.logical_and(0 < rois_area, + rois_area < area_threshold['P3'])] # pad a virtual rois if on rois assigned if rois_p5.size == 0: - rois_p5 = np.array([[12,34,56,78]]) + rois_p5 = np.array([[12, 34, 56, 78]]) if rois_p4.size == 0: - rois_p4 = np.array([[12,34,56,78]]) + rois_p4 = np.array([[12, 34, 56, 78]]) if rois_p3.size == 0: - rois_p3 = np.array([[12,34,56,78]]) + rois_p3 = np.array([[12, 34, 56, 78]]) if rois_p2.size == 0: - rois_p2 = np.array([[12,34,56,78]]) + rois_p2 = np.array([[12, 34, 56, 78]]) p5_batch_index = 0 * np.ones((rois_p5.shape[0], 1)) rois_p5_array = np.hstack((p5_batch_index, rois_p5))[np.newaxis, :] @@ -220,28 +237,32 @@ def get_fpn_rcnn_testbatch(roidb): p2_batch_index = 0 * np.ones((rois_p2.shape[0], 1)) rois_p2_array = np.hstack((p2_batch_index, rois_p2))[np.newaxis, :] - data = {'data': im_array, - 'rois_stride32': rois_p5_array, - 'rois_stride16': rois_p4_array, - 'rois_stride8': rois_p3_array, - 'rois_stride4': rois_p2_array} + data = { + 'data': im_array, + 'rois_stride32': rois_p5_array, + 'rois_stride16': rois_p4_array, + 'rois_stride8': rois_p3_array, + 'rois_stride4': rois_p2_array + } label = {} return data, label, im_info + def get_fpn_maskrcnn_batch(roidb): """ return a dictionary that contains raw data. """ num_images = len(roidb) - imgs, roidb = get_image(roidb, scale=config.TRAIN.SCALE) #TODO + imgs, roidb = get_image(roidb, scale=config.TRAIN.SCALE) #TODO #imgs, roidb = get_image(roidb) im_array = tensor_vstack(imgs) assert config.TRAIN.BATCH_ROIS % config.TRAIN.BATCH_IMAGES == 0, \ 'BATCHIMAGES {} must divide BATCH_ROIS {}'.format(config.TRAIN.BATCH_IMAGES, config.TRAIN.BATCH_ROIS) rois_per_image = config.TRAIN.BATCH_ROIS / config.TRAIN.BATCH_IMAGES - fg_rois_per_image = np.round(config.TRAIN.FG_FRACTION * rois_per_image).astype(int) + fg_rois_per_image = np.round(config.TRAIN.FG_FRACTION * + rois_per_image).astype(int) rois_on_imgs = dict() labels_on_imgs = dict() @@ -250,12 +271,12 @@ def get_fpn_maskrcnn_batch(roidb): mask_targets_on_imgs = dict() mask_weights_on_imgs = dict() for s in config.RCNN_FEAT_STRIDE: - rois_on_imgs.update({'stride%s' % s : list()}) - labels_on_imgs.update({'stride%s' % s : list()}) - bbox_targets_on_imgs.update({'stride%s' % s : list()}) - bbox_weights_on_imgs.update({'stride%s' % s : list()}) - mask_targets_on_imgs.update({'stride%s' % s : list()}) - mask_weights_on_imgs.update({'stride%s' % s : list()}) + rois_on_imgs.update({'stride%s' % s: list()}) + labels_on_imgs.update({'stride%s' % s: list()}) + bbox_targets_on_imgs.update({'stride%s' % s: list()}) + bbox_weights_on_imgs.update({'stride%s' % s: list()}) + mask_targets_on_imgs.update({'stride%s' % s: list()}) + mask_weights_on_imgs.update({'stride%s' % s: list()}) # Sample rois level_related_data_on_imgs = {} @@ -280,19 +301,31 @@ def get_fpn_maskrcnn_batch(roidb): sample_rois_fpn(rois, assign_levels, fg_rois_per_image, rois_per_image, num_classes, labels, overlaps, bbox_targets, mask_targets=mask_targets, mask_labels=mask_labels, mask_inds=mask_inds, im_info=im_info) - level_related_data_on_imgs.update({'img_%s' % im_i: {'rois_on_levels': im_rois_on_levels, - 'labels_on_levels': labels_on_levels, - 'bbox_targets_on_levels': bbox_targets_on_levels, - 'bbox_weights_on_levels': bbox_weights_on_levels, - 'mask_targets_on_levels': mask_targets_on_levels, - 'mask_weights_on_levels': mask_weights_on_levels, }}) + level_related_data_on_imgs.update({ + 'img_%s' % im_i: { + 'rois_on_levels': im_rois_on_levels, + 'labels_on_levels': labels_on_levels, + 'bbox_targets_on_levels': bbox_targets_on_levels, + 'bbox_weights_on_levels': bbox_weights_on_levels, + 'mask_targets_on_levels': mask_targets_on_levels, + 'mask_weights_on_levels': mask_weights_on_levels, + } + }) return im_array, level_related_data_on_imgs -def sample_rois(rois, fg_rois_per_image, rois_per_image, num_classes, - labels=None, overlaps=None, bbox_targets=None, gt_boxes=None, mask_targets=None, - mask_labels=None, mask_inds=None): +def sample_rois(rois, + fg_rois_per_image, + rois_per_image, + num_classes, + labels=None, + overlaps=None, + bbox_targets=None, + gt_boxes=None, + mask_targets=None, + mask_labels=None, + mask_inds=None): """ generate random sample of ROIs comprising foreground and background examples :param rois: all_rois [n, 4]; e2e: [n, 5] with batch_index @@ -312,7 +345,8 @@ def sample_rois(rois, fg_rois_per_image, rois_per_image, num_classes, overlaps = np.zeros((len(rois), )) labels = np.zeros((len(rois), )) else: - overlaps = bbox_overlaps(rois[:, 1:].astype(np.float), gt_boxes[:, :4].astype(np.float)) + overlaps = bbox_overlaps(rois[:, 1:].astype(np.float), + gt_boxes[:, :4].astype(np.float)) gt_assignment = overlaps.argmax(axis=1) overlaps = overlaps.max(axis=1) labels = gt_boxes[gt_assignment, 4] @@ -324,16 +358,22 @@ def sample_rois(rois, fg_rois_per_image, rois_per_image, num_classes, fg_rois_per_this_image = np.minimum(fg_rois_per_image, fg_indexes.size) # Sample foreground regions without replacement if len(fg_indexes) > fg_rois_per_this_image: - fg_indexes = npr.choice(fg_indexes, size=fg_rois_per_this_image, replace=False) + fg_indexes = npr.choice(fg_indexes, + size=fg_rois_per_this_image, + replace=False) # Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI) - bg_indexes = np.where((overlaps < config.TRAIN.BG_THRESH_HI) & (overlaps >= config.TRAIN.BG_THRESH_LO))[0] + bg_indexes = np.where((overlaps < config.TRAIN.BG_THRESH_HI) + & (overlaps >= config.TRAIN.BG_THRESH_LO))[0] # Compute number of background RoIs to take from this image (guarding against there being fewer than desired) bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image - bg_rois_per_this_image = np.minimum(bg_rois_per_this_image, bg_indexes.size) + bg_rois_per_this_image = np.minimum(bg_rois_per_this_image, + bg_indexes.size) # Sample foreground regions without replacement if len(bg_indexes) > bg_rois_per_this_image: - bg_indexes = npr.choice(bg_indexes, size=bg_rois_per_this_image, replace=False) + bg_indexes = npr.choice(bg_indexes, + size=bg_rois_per_this_image, + replace=False) # indexes selected keep_indexes = np.append(fg_indexes, bg_indexes) @@ -355,14 +395,19 @@ def sample_rois(rois, fg_rois_per_image, rois_per_image, num_classes, if mask_targets is not None: assert mask_labels is not None assert mask_inds is not None + def _mask_umap(mask_targets, mask_labels, mask_inds): - _mask_targets = np.zeros((num_rois, num_classes, 28, 28), dtype=np.int8) - _mask_weights = np.zeros((num_rois, num_classes, 28, 28), dtype=np.int8) + _mask_targets = np.zeros((num_rois, num_classes, 28, 28), + dtype=np.int8) + _mask_weights = np.zeros((num_rois, num_classes, 28, 28), + dtype=np.int8) _mask_targets[mask_inds, mask_labels] = mask_targets _mask_weights[mask_inds, mask_labels] = 1 - _mask_weights[:, 0] = 0 # set background mask weight to zeros - return _mask_targets, _mask_weights # [num_rois, num_classes, 28, 28] - mask_targets, mask_weights = _mask_umap(mask_targets, mask_labels, mask_inds) + _mask_weights[:, 0] = 0 # set background mask weight to zeros + return _mask_targets, _mask_weights # [num_rois, num_classes, 28, 28] + + mask_targets, mask_weights = _mask_umap(mask_targets, mask_labels, + mask_inds) mask_targets = mask_targets[keep_indexes] mask_weights = mask_weights[keep_indexes] @@ -370,22 +415,35 @@ def sample_rois(rois, fg_rois_per_image, rois_per_image, num_classes, if bbox_targets is not None: bbox_target_data = bbox_targets[keep_indexes, :] else: - targets = bbox_transform(rois[:, 1:], gt_boxes[gt_assignment[keep_indexes], :4]) + targets = bbox_transform(rois[:, 1:], + gt_boxes[gt_assignment[keep_indexes], :4]) if config.TRAIN.BBOX_NORMALIZATION_PRECOMPUTED: - targets = ((targets - np.array(config.TRAIN.BBOX_MEANS)) - / np.array(config.TRAIN.BBOX_STDS)) + targets = ((targets - np.array(config.TRAIN.BBOX_MEANS)) / + np.array(config.TRAIN.BBOX_STDS)) bbox_target_data = np.hstack((labels[:, np.newaxis], targets)) bbox_targets, bbox_weights = \ expand_bbox_regression_targets(bbox_target_data, num_classes) - + if mask_targets is not None: return rois, labels, bbox_targets, bbox_weights, mask_targets, mask_weights else: return rois, labels, bbox_targets, bbox_weights -def sample_rois_fpn(rois, assign_levels, fg_rois_per_image, rois_per_image, num_classes, - labels=None, overlaps=None, bbox_targets=None, mask_targets=None, mask_labels=None, mask_inds=None, gt_boxes=None, im_info=None): + +def sample_rois_fpn(rois, + assign_levels, + fg_rois_per_image, + rois_per_image, + num_classes, + labels=None, + overlaps=None, + bbox_targets=None, + mask_targets=None, + mask_labels=None, + mask_inds=None, + gt_boxes=None, + im_info=None): """ generate random sample of ROIs comprising foreground and background examples :param rois: all_rois [n, 4]; e2e: [n, 5] with batch_index @@ -407,7 +465,8 @@ def sample_rois_fpn(rois, assign_levels, fg_rois_per_image, rois_per_image, num_ overlaps = np.zeros((len(rois), )) labels = np.zeros((len(rois), )) else: - overlaps = bbox_overlaps(rois[:, 1:].astype(np.float), gt_boxes[:, :4].astype(np.float)) + overlaps = bbox_overlaps(rois[:, 1:].astype(np.float), + gt_boxes[:, :4].astype(np.float)) gt_assignment = overlaps.argmax(axis=1) overlaps = overlaps.max(axis=1) labels = gt_boxes[gt_assignment, 4] @@ -423,18 +482,24 @@ def sample_rois_fpn(rois, assign_levels, fg_rois_per_image, rois_per_image, num_ # Sample foreground regions without replacement if len(fg_indexes) > fg_rois_per_this_image: - fg_indexes = npr.choice(fg_indexes, size=fg_rois_per_this_image, replace=False) + fg_indexes = npr.choice(fg_indexes, + size=fg_rois_per_this_image, + replace=False) # Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI) - bg_indexes = np.where((overlaps < config.TRAIN.BG_THRESH_HI) & (overlaps >= config.TRAIN.BG_THRESH_LO))[0] + bg_indexes = np.where((overlaps < config.TRAIN.BG_THRESH_HI) + & (overlaps >= config.TRAIN.BG_THRESH_LO))[0] if DEBUG: print 'bg total num:', len(bg_indexes) # Compute number of background RoIs to take from this image (guarding against there being fewer than desired) bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image - bg_rois_per_this_image = np.minimum(bg_rois_per_this_image, bg_indexes.size) + bg_rois_per_this_image = np.minimum(bg_rois_per_this_image, + bg_indexes.size) # Sample foreground regions without replacement if len(bg_indexes) > bg_rois_per_this_image: - bg_indexes = npr.choice(bg_indexes, size=bg_rois_per_this_image, replace=False) + bg_indexes = npr.choice(bg_indexes, + size=bg_rois_per_this_image, + replace=False) if DEBUG: print 'fg num:', len(fg_indexes) print 'bg num:', len(bg_indexes) @@ -444,7 +509,8 @@ def sample_rois_fpn(rois, assign_levels, fg_rois_per_image, rois_per_image, num_ bg_assign = assign_levels[bg_indexes] bg_rois_on_levels = dict() for i, s in enumerate(config.RCNN_FEAT_STRIDE): - bg_rois_on_levels.update({'stride%s'%s:len(np.where(bg_assign == s)[0])}) + bg_rois_on_levels.update( + {'stride%s' % s: len(np.where(bg_assign == s)[0])}) print bg_rois_on_levels # indexes selected @@ -469,13 +535,18 @@ def sample_rois_fpn(rois, assign_levels, fg_rois_per_image, rois_per_image, num_ if mask_targets is not None: assert mask_labels is not None assert mask_inds is not None + def _mask_umap(mask_targets, mask_labels, mask_inds): - _mask_targets = np.zeros((num_rois, num_classes, 28, 28), dtype=np.int8) - _mask_weights = np.zeros((num_rois, num_classes, 1, 1), dtype=np.int8) + _mask_targets = np.zeros((num_rois, num_classes, 28, 28), + dtype=np.int8) + _mask_weights = np.zeros((num_rois, num_classes, 1, 1), + dtype=np.int8) _mask_targets[mask_inds, mask_labels] = mask_targets _mask_weights[mask_inds, mask_labels] = 1 - return _mask_targets, _mask_weights # [num_rois, num_classes, 28, 28] - mask_targets, mask_weights = _mask_umap(mask_targets, mask_labels, mask_inds) + return _mask_targets, _mask_weights # [num_rois, num_classes, 28, 28] + + mask_targets, mask_weights = _mask_umap(mask_targets, mask_labels, + mask_inds) mask_targets = mask_targets[keep_indexes] mask_weights = mask_weights[keep_indexes] @@ -483,10 +554,11 @@ def sample_rois_fpn(rois, assign_levels, fg_rois_per_image, rois_per_image, num_ if bbox_targets is not None: bbox_target_data = bbox_targets[keep_indexes, :] else: - targets = bbox_transform(rois[:, 1:], gt_boxes[gt_assignment[keep_indexes], :4]) + targets = bbox_transform(rois[:, 1:], + gt_boxes[gt_assignment[keep_indexes], :4]) if config.TRAIN.BBOX_NORMALIZATION_PRECOMPUTED: - targets = ((targets - np.array(config.TRAIN.BBOX_MEANS)) - / np.array(config.TRAIN.BBOX_STDS)) + targets = ((targets - np.array(config.TRAIN.BBOX_MEANS)) / + np.array(config.TRAIN.BBOX_STDS)) bbox_target_data = np.hstack((labels[:, np.newaxis], targets)) bbox_targets, bbox_weights = \ @@ -502,8 +574,8 @@ def sample_rois_fpn(rois, assign_levels, fg_rois_per_image, rois_per_image, num_ mask_weights_on_levels = dict() for i, s in enumerate(config.RCNN_FEAT_STRIDE): index = np.where(assign_levels == s) - _rois = rois[index] - _labels = labels[index] + _rois = rois[index] + _labels = labels[index] _bbox_targets = bbox_targets[index] _bbox_weights = bbox_weights[index] if mask_targets is not None: @@ -519,12 +591,18 @@ def sample_rois_fpn(rois, assign_levels, fg_rois_per_image, rois_per_image, num_ mask_weights_on_levels.update({'stride%s' % s: _mask_weights}) if mask_targets is not None: - return rois_on_levels, labels_on_levels, bbox_targets_on_levels, bbox_weights_on_levels,mask_targets_on_levels,mask_weights_on_levels + return rois_on_levels, labels_on_levels, bbox_targets_on_levels, bbox_weights_on_levels, mask_targets_on_levels, mask_weights_on_levels else: return rois_on_levels, labels_on_levels, bbox_targets_on_levels, bbox_weights_on_levels -def get_rois(rois, rois_per_image, num_classes, - labels=None, overlaps=None, bbox_targets=None, gt_boxes=None): + +def get_rois(rois, + rois_per_image, + num_classes, + labels=None, + overlaps=None, + bbox_targets=None, + gt_boxes=None): """ get top N ROIs, used in online hard example mining :param rois: all_rois [n, 4]; e2e: [n, 5] with batch_index @@ -539,7 +617,8 @@ def get_rois(rois, rois_per_image, num_classes, if labels is None: if len(gt_boxes) == 0: gt_boxes = np.array([[1, 1, 1, 1, 0]]) - overlaps = bbox_overlaps(rois[:, 1:].astype(np.float), gt_boxes[:, :4].astype(np.float)) + overlaps = bbox_overlaps(rois[:, 1:].astype(np.float), + gt_boxes[:, :4].astype(np.float)) gt_assignment = overlaps.argmax(axis=1) overlaps = overlaps.max(axis=1) labels = gt_boxes[gt_assignment, 4] @@ -547,7 +626,9 @@ def get_rois(rois, rois_per_image, num_classes, # select indices keep_indexes = np.arange(rois.shape[0]) if keep_indexes.shape[0] > rois_per_image: - keep_indexes = npr.choice(keep_indexes, size=rois_per_image, replace=False) + keep_indexes = npr.choice(keep_indexes, + size=rois_per_image, + replace=False) # if not enough, pad until rois_per_image is satisfied while keep_indexes.shape[0] < rois_per_image: @@ -556,7 +637,8 @@ def get_rois(rois, rois_per_image, num_classes, keep_indexes = np.append(keep_indexes, gap_indexes) # suppress any bg defined by overlap - bg_indexes = np.where((overlaps < config.TRAIN.BG_THRESH_HI) & (overlaps >= config.TRAIN.BG_THRESH_LO))[0] + bg_indexes = np.where((overlaps < config.TRAIN.BG_THRESH_HI) + & (overlaps >= config.TRAIN.BG_THRESH_LO))[0] labels[bg_indexes] = 0 labels = labels[keep_indexes] @@ -566,14 +648,14 @@ def get_rois(rois, rois_per_image, num_classes, if bbox_targets is not None: bbox_target_data = bbox_targets[keep_indexes, :] else: - targets = bbox_transform(rois[:, 1:], gt_boxes[gt_assignment[keep_indexes], :4]) + targets = bbox_transform(rois[:, 1:], + gt_boxes[gt_assignment[keep_indexes], :4]) if config.TRAIN.BBOX_NORMALIZATION_PRECOMPUTED: - targets = ((targets - np.array(config.TRAIN.BBOX_MEANS)) - / np.array(config.TRAIN.BBOX_STDS)) + targets = ((targets - np.array(config.TRAIN.BBOX_MEANS)) / + np.array(config.TRAIN.BBOX_STDS)) bbox_target_data = np.hstack((labels[:, np.newaxis], targets)) bbox_targets, bbox_weights = \ expand_bbox_regression_targets(bbox_target_data, num_classes) return rois, labels, bbox_targets, bbox_weights - diff --git a/detection/RetinaFace/rcnn/io/rpn.py b/detection/RetinaFace/rcnn/io/rpn.py new file mode 100644 index 0000000..998fad8 --- /dev/null +++ b/detection/RetinaFace/rcnn/io/rpn.py @@ -0,0 +1,887 @@ +""" +RPN: +data = + {'data': [num_images, c, h, w], + 'im_info': [num_images, 4] (optional)} +label = + {'gt_boxes': [num_boxes, 5] (optional), + 'label': [batch_size, 1] <- [batch_size, num_anchors, feat_height, feat_width], + 'bbox_target': [batch_size, num_anchors, feat_height, feat_width], + 'bbox_weight': [batch_size, num_anchors, feat_height, feat_width]} +""" + +from __future__ import print_function +import sys +import logging +import datetime +import numpy as np +import numpy.random as npr + +from ..logger import logger +from ..config import config +from .image import get_image, tensor_vstack, get_crop_image +from ..processing.generate_anchor import generate_anchors, anchors_plane +from ..processing.bbox_transform import bbox_overlaps, bbox_transform, landmark_transform + +STAT = {0: 0, 8: 0, 16: 0, 32: 0} + + +def get_rpn_testbatch(roidb): + """ + return a dict of testbatch + :param roidb: ['image', 'flipped'] + :return: data, label, im_info + """ + assert len(roidb) == 1, 'Single batch only' + imgs, roidb = get_image(roidb) + im_array = imgs[0] + im_info = np.array([roidb[0]['im_info']], dtype=np.float32) + + data = {'data': im_array, 'im_info': im_info} + label = {} + + return data, label, im_info + + +def get_rpn_batch(roidb): + """ + prototype for rpn batch: data, im_info, gt_boxes + :param roidb: ['image', 'flipped'] + ['gt_boxes', 'boxes', 'gt_classes'] + :return: data, label + """ + assert len(roidb) == 1, 'Single batch only' + imgs, roidb = get_image(roidb) + im_array = imgs[0] + im_info = np.array([roidb[0]['im_info']], dtype=np.float32) + + # gt boxes: (x1, y1, x2, y2, cls) + if roidb[0]['gt_classes'].size > 0: + gt_inds = np.where(roidb[0]['gt_classes'] != 0)[0] + gt_boxes = np.empty((roidb[0]['boxes'].shape[0], 5), dtype=np.float32) + gt_boxes[:, 0:4] = roidb[0]['boxes'][gt_inds, :] + gt_boxes[:, 4] = roidb[0]['gt_classes'][gt_inds] + else: + gt_boxes = np.empty((0, 5), dtype=np.float32) + + data = {'data': im_array, 'im_info': im_info} + label = {'gt_boxes': gt_boxes} + + return data, label + + +def get_crop_batch(roidb): + """ + prototype for rpn batch: data, im_info, gt_boxes + :param roidb: ['image', 'flipped'] + ['gt_boxes', 'boxes', 'gt_classes'] + :return: data, label + """ + #assert len(roidb) == 1, 'Single batch only' + data_list = [] + label_list = [] + imgs, roidb = get_crop_image(roidb) + assert len(imgs) == len(roidb) + for i in range(len(imgs)): + im_array = imgs[i] + im_info = np.array([roidb[i]['im_info']], dtype=np.float32) + + # gt boxes: (x1, y1, x2, y2, cls) + if roidb[i]['gt_classes'].size > 0: + gt_inds = np.where(roidb[i]['gt_classes'] != 0)[0] + gt_boxes = np.empty((roidb[i]['boxes'].shape[0], 5), + dtype=np.float32) + gt_boxes[:, 0:4] = roidb[i]['boxes'][gt_inds, :] + gt_boxes[:, 4] = roidb[i]['gt_classes'][gt_inds] + if config.USE_BLUR: + gt_blur = roidb[i]['blur'] + if config.FACE_LANDMARK: + #gt_landmarks = np.empty((roidb[i]['landmarks'].shape[0], 11), dtype=np.float32) + gt_landmarks = roidb[i]['landmarks'][gt_inds, :, :] + if config.HEAD_BOX: + gt_boxes_head = np.empty((roidb[i]['boxes_head'].shape[0], 5), + dtype=np.float32) + gt_boxes_head[:, 0:4] = roidb[i]['boxes_head'][gt_inds, :] + gt_boxes_head[:, 4] = roidb[i]['gt_classes'][gt_inds] + else: + gt_boxes = np.empty((0, 5), dtype=np.float32) + if config.USE_BLUR: + gt_blur = np.empty((0, ), dtype=np.float32) + if config.FACE_LANDMARK: + gt_landmarks = np.empty((0, 5, 3), dtype=np.float32) + if config.HEAD_BOX: + gt_boxes_head = np.empty((0, 5), dtype=np.float32) + + data = {'data': im_array, 'im_info': im_info} + label = {'gt_boxes': gt_boxes} + if config.USE_BLUR: + label['gt_blur'] = gt_blur + if config.FACE_LANDMARK: + label['gt_landmarks'] = gt_landmarks + if config.HEAD_BOX: + label['gt_boxes_head'] = gt_boxes_head + data_list.append(data) + label_list.append(label) + + return data_list, label_list + + +def assign_anchor_fpn(feat_shape, + gt_label, + im_info, + landmark=False, + prefix='face', + select_stride=0): + """ + assign ground truth boxes to anchor positions + :param feat_shape: infer output shape + :param gt_boxes: assign ground truth + :param im_info: filter out anchors overlapped with edges + :return: tuple + labels: of shape (batch_size, 1) <- (batch_size, num_anchors, feat_height, feat_width) + bbox_targets: of shape (batch_size, num_anchors * 4, feat_height, feat_width) + bbox_weights: mark the assigned anchors + """ + def _unmap(data, count, inds, fill=0): + """" unmap a subset inds of data into original data of size count """ + if len(data.shape) == 1: + ret = np.empty((count, ), dtype=np.float32) + ret.fill(fill) + ret[inds] = data + else: + ret = np.empty((count, ) + data.shape[1:], dtype=np.float32) + ret.fill(fill) + ret[inds, :] = data + return ret + + global STAT + DEBUG = False + + im_info = im_info[0] + gt_boxes = gt_label['gt_boxes'] + # clean up boxes + nonneg = np.where(gt_boxes[:, 4] != -1)[0] + gt_boxes = gt_boxes[nonneg] + if config.USE_BLUR: + gt_blur = gt_label['gt_blur'] + gt_blur = gt_blur[nonneg] + if landmark: + gt_landmarks = gt_label['gt_landmarks'] + gt_landmarks = gt_landmarks[nonneg] + assert gt_boxes.shape[0] == gt_landmarks.shape[0] + #scales = np.array(scales, dtype=np.float32) + feat_strides = config.RPN_FEAT_STRIDE + bbox_pred_len = 4 + landmark_pred_len = 10 + if config.USE_BLUR: + gt_boxes[:, 4] = gt_blur + bbox_pred_len = 5 + if config.USE_OCCLUSION: + landmark_pred_len = 15 + + anchors_list = [] + anchors_num_list = [] + inds_inside_list = [] + feat_infos = [] + A_list = [] + for i in range(len(feat_strides)): + stride = feat_strides[i] + sstride = str(stride) + base_size = config.RPN_ANCHOR_CFG[sstride]['BASE_SIZE'] + allowed_border = config.RPN_ANCHOR_CFG[sstride]['ALLOWED_BORDER'] + ratios = config.RPN_ANCHOR_CFG[sstride]['RATIOS'] + scales = config.RPN_ANCHOR_CFG[sstride]['SCALES'] + base_anchors = generate_anchors(base_size=base_size, + ratios=list(ratios), + scales=np.array(scales, + dtype=np.float32), + stride=stride, + dense_anchor=config.DENSE_ANCHOR) + num_anchors = base_anchors.shape[0] + feat_height, feat_width = feat_shape[i][-2:] + feat_stride = feat_strides[i] + feat_infos.append([feat_height, feat_width]) + + A = num_anchors + A_list.append(A) + K = feat_height * feat_width + + all_anchors = anchors_plane(feat_height, feat_width, feat_stride, + base_anchors) + all_anchors = all_anchors.reshape((K * A, 4)) + #print('anchor0', stride, all_anchors[0]) + + total_anchors = int(K * A) + anchors_num_list.append(total_anchors) + # only keep anchors inside the image + inds_inside = np.where( + (all_anchors[:, 0] >= -allowed_border) + & (all_anchors[:, 1] >= -allowed_border) + & (all_anchors[:, 2] < im_info[1] + allowed_border) + & (all_anchors[:, 3] < im_info[0] + allowed_border))[0] + if DEBUG: + print('total_anchors', total_anchors) + print('inds_inside', len(inds_inside)) + + # keep only inside anchors + anchors = all_anchors[inds_inside, :] + #print('AA', anchors.shape, len(inds_inside)) + + anchors_list.append(anchors) + inds_inside_list.append(inds_inside) + + # Concat anchors from each level + anchors = np.concatenate(anchors_list) + for i in range(1, len(inds_inside_list)): + inds_inside_list[i] = inds_inside_list[i] + sum(anchors_num_list[:i]) + inds_inside = np.concatenate(inds_inside_list) + total_anchors = sum(anchors_num_list) + #print('total_anchors', anchors.shape[0], len(inds_inside), file=sys.stderr) + + # label: 1 is positive, 0 is negative, -1 is dont care + labels = np.empty((len(inds_inside), ), dtype=np.float32) + labels.fill(-1) + #print('BB', anchors.shape, len(inds_inside)) + #print('gt_boxes', gt_boxes.shape, file=sys.stderr) + + if gt_boxes.size > 0: + # overlap between the anchors and the gt boxes + # overlaps (ex, gt) + overlaps = bbox_overlaps(anchors.astype(np.float), + gt_boxes.astype(np.float)) + argmax_overlaps = overlaps.argmax(axis=1) + #print('AAA', argmax_overlaps.shape) + max_overlaps = overlaps[np.arange(len(inds_inside)), argmax_overlaps] + gt_argmax_overlaps = overlaps.argmax(axis=0) + gt_max_overlaps = overlaps[gt_argmax_overlaps, + np.arange(overlaps.shape[1])] + gt_argmax_overlaps = np.where(overlaps == gt_max_overlaps)[0] + + if not config.TRAIN.RPN_CLOBBER_POSITIVES: + # assign bg labels first so that positive labels can clobber them + labels[max_overlaps < config.TRAIN.RPN_NEGATIVE_OVERLAP] = 0 + + # fg label: for each gt, anchor with highest overlap + if config.TRAIN.RPN_FORCE_POSITIVE: + labels[gt_argmax_overlaps] = 1 + + # fg label: above threshold IoU + labels[max_overlaps >= config.TRAIN.RPN_POSITIVE_OVERLAP] = 1 + + if config.TRAIN.RPN_CLOBBER_POSITIVES: + # assign bg labels last so that negative labels can clobber positives + labels[max_overlaps < config.TRAIN.RPN_NEGATIVE_OVERLAP] = 0 + else: + labels[:] = 0 + fg_inds = np.where(labels == 1)[0] + #print('fg count', len(fg_inds)) + + # subsample positive labels if we have too many + if config.TRAIN.RPN_ENABLE_OHEM == 0: + fg_inds = np.where(labels == 1)[0] + num_fg = int(config.TRAIN.RPN_FG_FRACTION * + config.TRAIN.RPN_BATCH_SIZE) + if len(fg_inds) > num_fg: + disable_inds = npr.choice(fg_inds, + size=(len(fg_inds) - num_fg), + replace=False) + if DEBUG: + disable_inds = fg_inds[:(len(fg_inds) - num_fg)] + labels[disable_inds] = -1 + + # subsample negative labels if we have too many + num_bg = config.TRAIN.RPN_BATCH_SIZE - np.sum(labels == 1) + bg_inds = np.where(labels == 0)[0] + if len(bg_inds) > num_bg: + disable_inds = npr.choice(bg_inds, + size=(len(bg_inds) - num_bg), + replace=False) + if DEBUG: + disable_inds = bg_inds[:(len(bg_inds) - num_bg)] + labels[disable_inds] = -1 + + #fg_inds = np.where(labels == 1)[0] + #num_fg = len(fg_inds) + #num_bg = num_fg*int(1.0/config.TRAIN.RPN_FG_FRACTION-1) + + #bg_inds = np.where(labels == 0)[0] + #if len(bg_inds) > num_bg: + # disable_inds = npr.choice(bg_inds, size=(len(bg_inds) - num_bg), replace=False) + # if DEBUG: + # disable_inds = bg_inds[:(len(bg_inds) - num_bg)] + # labels[disable_inds] = -1 + else: + fg_inds = np.where(labels == 1)[0] + num_fg = len(fg_inds) + bg_inds = np.where(labels == 0)[0] + num_bg = len(bg_inds) + + #print('anchor stat', num_fg, num_bg) + + bbox_targets = np.zeros((len(inds_inside), bbox_pred_len), + dtype=np.float32) + if gt_boxes.size > 0: + #print('GT', gt_boxes.shape, gt_boxes[argmax_overlaps, :4].shape) + bbox_targets[:, :] = bbox_transform(anchors, + gt_boxes[argmax_overlaps, :]) + #bbox_targets[:,4] = gt_blur + + bbox_weights = np.zeros((len(inds_inside), bbox_pred_len), + dtype=np.float32) + #bbox_weights[labels == 1, :] = np.array(config.TRAIN.RPN_BBOX_WEIGHTS) + bbox_weights[labels == 1, 0:4] = 1.0 + if bbox_pred_len > 4: + bbox_weights[labels == 1, 4:bbox_pred_len] = 0.1 + + if landmark: + landmark_targets = np.zeros((len(inds_inside), landmark_pred_len), + dtype=np.float32) + #landmark_weights = np.zeros((len(inds_inside), 10), dtype=np.float32) + landmark_weights = np.zeros((len(inds_inside), landmark_pred_len), + dtype=np.float32) + #landmark_weights[labels == 1, :] = np.array(config.TRAIN.RPN_LANDMARK_WEIGHTS) + if landmark_pred_len == 10: + landmark_weights[labels == 1, :] = 1.0 + elif landmark_pred_len == 15: + v = [1.0, 1.0, 0.1] * 5 + assert len(v) == 15 + landmark_weights[labels == 1, :] = np.array(v) + else: + assert False + #TODO here + if gt_landmarks.size > 0: + #print('AAA',argmax_overlaps) + a_landmarks = gt_landmarks[argmax_overlaps, :, :] + landmark_targets[:] = landmark_transform(anchors, a_landmarks) + invalid = np.where(a_landmarks[:, 0, 2] < 0.0)[0] + #assert len(invalid)==0 + #landmark_weights[invalid, :] = np.array(config.TRAIN.RPN_INVALID_LANDMARK_WEIGHTS) + landmark_weights[invalid, :] = 0.0 + + #if DEBUG: + # _sums = bbox_targets[labels == 1, :].sum(axis=0) + # _squared_sums = (bbox_targets[labels == 1, :] ** 2).sum(axis=0) + # _counts = np.sum(labels == 1) + # means = _sums / (_counts + 1e-14) + # stds = np.sqrt(_squared_sums / _counts - means ** 2) + # print 'means', means + # print 'stdevs', stds + # map up to original set of anchors + #print(labels.shape, total_anchors, inds_inside.shape, inds_inside[0], inds_inside[-1]) + labels = _unmap(labels, total_anchors, inds_inside, fill=-1) + bbox_targets = _unmap(bbox_targets, total_anchors, inds_inside, fill=0) + bbox_weights = _unmap(bbox_weights, total_anchors, inds_inside, fill=0) + if landmark: + landmark_targets = _unmap(landmark_targets, + total_anchors, + inds_inside, + fill=0) + landmark_weights = _unmap(landmark_weights, + total_anchors, + inds_inside, + fill=0) + #print('CC', anchors.shape, len(inds_inside)) + + #if DEBUG: + # if gt_boxes.size > 0: + # print 'rpn: max max_overlaps', np.max(max_overlaps) + # print 'rpn: num_positives', np.sum(labels == 1) + # print 'rpn: num_negatives', np.sum(labels == 0) + # _fg_sum = np.sum(labels == 1) + # _bg_sum = np.sum(labels == 0) + # _count = 1 + # print 'rpn: num_positive avg', _fg_sum / _count + # print 'rpn: num_negative avg', _bg_sum / _count + + # resahpe + label_list = list() + bbox_target_list = list() + bbox_weight_list = list() + if landmark: + landmark_target_list = list() + landmark_weight_list = list() + anchors_num_range = [0] + anchors_num_list + label = {} + for i in range(len(feat_strides)): + stride = feat_strides[i] + feat_height, feat_width = feat_infos[i] + A = A_list[i] + _label = labels[sum(anchors_num_range[:i + + 1]):sum(anchors_num_range[:i + + 1]) + + anchors_num_range[i + 1]] + if select_stride > 0 and stride != select_stride: + #print('set', stride, select_stride) + _label[:] = -1 + #print('_label', _label.shape, select_stride) + #_fg_inds = np.where(_label == 1)[0] + #n_fg = len(_fg_inds) + #STAT[0]+=1 + #STAT[stride]+=n_fg + #if STAT[0]%100==0: + # print('rpn_stat', STAT, file=sys.stderr) + bbox_target = bbox_targets[sum(anchors_num_range[:i + 1] + ):sum(anchors_num_range[:i + 1]) + + anchors_num_range[i + 1]] + bbox_weight = bbox_weights[sum(anchors_num_range[:i + 1] + ):sum(anchors_num_range[:i + 1]) + + anchors_num_range[i + 1]] + if landmark: + landmark_target = landmark_targets[ + sum(anchors_num_range[:i + 1]):sum(anchors_num_range[:i + 1]) + + anchors_num_range[i + 1]] + landmark_weight = landmark_weights[ + sum(anchors_num_range[:i + 1]):sum(anchors_num_range[:i + 1]) + + anchors_num_range[i + 1]] + + _label = _label.reshape( + (1, feat_height, feat_width, A)).transpose(0, 3, 1, 2) + _label = _label.reshape((1, A * feat_height * feat_width)) + bbox_target = bbox_target.reshape( + (1, feat_height * feat_width, + A * bbox_pred_len)).transpose(0, 2, 1) + bbox_weight = bbox_weight.reshape( + (1, feat_height * feat_width, A * bbox_pred_len)).transpose( + (0, 2, 1)) + label['%s_label_stride%d' % (prefix, stride)] = _label + label['%s_bbox_target_stride%d' % (prefix, stride)] = bbox_target + label['%s_bbox_weight_stride%d' % (prefix, stride)] = bbox_weight + if landmark: + landmark_target = landmark_target.reshape( + (1, feat_height * feat_width, + A * landmark_pred_len)).transpose(0, 2, 1) + landmark_weight = landmark_weight.reshape( + (1, feat_height * feat_width, + A * landmark_pred_len)).transpose((0, 2, 1)) + label['%s_landmark_target_stride%d' % + (prefix, stride)] = landmark_target + label['%s_landmark_weight_stride%d' % + (prefix, stride)] = landmark_weight + #print('in_rpn', stride,_label.shape, bbox_target.shape, bbox_weight.shape, file=sys.stderr) + label_list.append(_label) + #print('DD', _label.shape) + bbox_target_list.append(bbox_target) + bbox_weight_list.append(bbox_weight) + if landmark: + landmark_target_list.append(landmark_target) + landmark_weight_list.append(landmark_weight) + + label_concat = np.concatenate(label_list, axis=1) + bbox_target_concat = np.concatenate(bbox_target_list, axis=2) + bbox_weight_concat = np.concatenate(bbox_weight_list, axis=2) + #fg_inds = np.where(label_concat[0] == 1)[0] + #print('fg_inds_in_rpn2', fg_inds, file=sys.stderr) + + label.update({ + '%s_label' % prefix: label_concat, + '%s_bbox_target' % prefix: bbox_target_concat, + '%s_bbox_weight' % prefix: bbox_weight_concat + }) + if landmark: + landmark_target_concat = np.concatenate(landmark_target_list, axis=2) + landmark_weight_concat = np.concatenate(landmark_weight_list, axis=2) + label['%s_landmark_target' % prefix] = landmark_target_concat + label['%s_landmark_weight' % prefix] = landmark_weight_concat + return label + + +class AA: + def __init__(self, feat_shape): + self.feat_shape = feat_shape + feat_strides = config.RPN_FEAT_STRIDE + anchors_list = [] + anchors_num_list = [] + inds_inside_list = [] + feat_infos = [] + A_list = [] + DEBUG = False + for i in range(len(feat_strides)): + stride = feat_strides[i] + sstride = str(stride) + base_size = config.RPN_ANCHOR_CFG[sstride]['BASE_SIZE'] + allowed_border = config.RPN_ANCHOR_CFG[sstride]['ALLOWED_BORDER'] + ratios = config.RPN_ANCHOR_CFG[sstride]['RATIOS'] + scales = config.RPN_ANCHOR_CFG[sstride]['SCALES'] + base_anchors = generate_anchors(base_size=base_size, + ratios=list(ratios), + scales=np.array(scales, + dtype=np.float32), + stride=stride, + dense_anchor=config.DENSE_ANCHOR) + num_anchors = base_anchors.shape[0] + feat_height, feat_width = feat_shape[i][-2:] + feat_stride = feat_strides[i] + feat_infos.append([feat_height, feat_width]) + + A = num_anchors + A_list.append(A) + K = feat_height * feat_width + + all_anchors = anchors_plane(feat_height, feat_width, feat_stride, + base_anchors) + all_anchors = all_anchors.reshape((K * A, 4)) + #print('anchor0', stride, all_anchors[0]) + + total_anchors = int(K * A) + anchors_num_list.append(total_anchors) + # only keep anchors inside the image + inds_inside = np.where( + (all_anchors[:, 0] >= -allowed_border) + & (all_anchors[:, 1] >= -allowed_border) + & (all_anchors[:, 2] < config.SCALES[0][1] + allowed_border) & + (all_anchors[:, 3] < config.SCALES[0][1] + allowed_border))[0] + if DEBUG: + print('total_anchors', total_anchors) + print('inds_inside', len(inds_inside)) + + # keep only inside anchors + anchors = all_anchors[inds_inside, :] + #print('AA', anchors.shape, len(inds_inside)) + + anchors_list.append(anchors) + inds_inside_list.append(inds_inside) + anchors = np.concatenate(anchors_list) + for i in range(1, len(inds_inside_list)): + inds_inside_list[i] = inds_inside_list[i] + sum( + anchors_num_list[:i]) + inds_inside = np.concatenate(inds_inside_list) + #self.anchors_list = anchors_list + #self.inds_inside_list = inds_inside_list + self.anchors = anchors + self.inds_inside = inds_inside + self.anchors_num_list = anchors_num_list + self.feat_infos = feat_infos + self.A_list = A_list + self._times = [0.0, 0.0, 0.0, 0.0] + + @staticmethod + def _unmap(data, count, inds, fill=0): + """" unmap a subset inds of data into original data of size count """ + if len(data.shape) == 1: + ret = np.empty((count, ), dtype=np.float32) + ret.fill(fill) + ret[inds] = data + else: + ret = np.empty((count, ) + data.shape[1:], dtype=np.float32) + ret.fill(fill) + ret[inds, :] = data + return ret + + def assign_anchor_fpn(self, + gt_label, + im_info, + landmark=False, + prefix='face', + select_stride=0): + + #ta = datetime.datetime.now() + im_info = im_info[0] + gt_boxes = gt_label['gt_boxes'] + # clean up boxes + nonneg = np.where(gt_boxes[:, 4] != -1)[0] + gt_boxes = gt_boxes[nonneg] + if config.USE_BLUR: + gt_blur = gt_label['gt_blur'] + gt_blur = gt_blur[nonneg] + if landmark: + gt_landmarks = gt_label['gt_landmarks'] + gt_landmarks = gt_landmarks[nonneg] + assert gt_boxes.shape[0] == gt_landmarks.shape[0] + #scales = np.array(scales, dtype=np.float32) + feat_strides = config.RPN_FEAT_STRIDE + bbox_pred_len = 4 + landmark_pred_len = 10 + if config.USE_BLUR: + gt_boxes[:, 4] = gt_blur + bbox_pred_len = 5 + if config.USE_OCCLUSION: + landmark_pred_len = 15 + + #anchors_list = self.anchors_list + #inds_inside_list = self.inds_inside_list + anchors = self.anchors + inds_inside = self.inds_inside + anchors_num_list = self.anchors_num_list + feat_infos = self.feat_infos + A_list = self.A_list + + total_anchors = sum(anchors_num_list) + #print('total_anchors', anchors.shape[0], len(inds_inside), file=sys.stderr) + + # label: 1 is positive, 0 is negative, -1 is dont care + labels = np.empty((len(inds_inside), ), dtype=np.float32) + labels.fill(-1) + #print('BB', anchors.shape, len(inds_inside)) + #print('gt_boxes', gt_boxes.shape, file=sys.stderr) + #tb = datetime.datetime.now() + #self._times[0] += (tb-ta).total_seconds() + #ta = datetime.datetime.now() + + if gt_boxes.size > 0: + # overlap between the anchors and the gt boxes + # overlaps (ex, gt) + overlaps = bbox_overlaps(anchors.astype(np.float), + gt_boxes.astype(np.float)) + argmax_overlaps = overlaps.argmax(axis=1) + #print('AAA', argmax_overlaps.shape) + max_overlaps = overlaps[np.arange(len(inds_inside)), + argmax_overlaps] + gt_argmax_overlaps = overlaps.argmax(axis=0) + gt_max_overlaps = overlaps[gt_argmax_overlaps, + np.arange(overlaps.shape[1])] + gt_argmax_overlaps = np.where(overlaps == gt_max_overlaps)[0] + + if not config.TRAIN.RPN_CLOBBER_POSITIVES: + # assign bg labels first so that positive labels can clobber them + labels[max_overlaps < config.TRAIN.RPN_NEGATIVE_OVERLAP] = 0 + + # fg label: for each gt, anchor with highest overlap + if config.TRAIN.RPN_FORCE_POSITIVE: + labels[gt_argmax_overlaps] = 1 + + # fg label: above threshold IoU + labels[max_overlaps >= config.TRAIN.RPN_POSITIVE_OVERLAP] = 1 + + if config.TRAIN.RPN_CLOBBER_POSITIVES: + # assign bg labels last so that negative labels can clobber positives + labels[max_overlaps < config.TRAIN.RPN_NEGATIVE_OVERLAP] = 0 + else: + labels[:] = 0 + fg_inds = np.where(labels == 1)[0] + #print('fg count', len(fg_inds)) + + # subsample positive labels if we have too many + if config.TRAIN.RPN_ENABLE_OHEM == 0: + fg_inds = np.where(labels == 1)[0] + num_fg = int(config.TRAIN.RPN_FG_FRACTION * + config.TRAIN.RPN_BATCH_SIZE) + if len(fg_inds) > num_fg: + disable_inds = npr.choice(fg_inds, + size=(len(fg_inds) - num_fg), + replace=False) + if DEBUG: + disable_inds = fg_inds[:(len(fg_inds) - num_fg)] + labels[disable_inds] = -1 + + # subsample negative labels if we have too many + num_bg = config.TRAIN.RPN_BATCH_SIZE - np.sum(labels == 1) + bg_inds = np.where(labels == 0)[0] + if len(bg_inds) > num_bg: + disable_inds = npr.choice(bg_inds, + size=(len(bg_inds) - num_bg), + replace=False) + if DEBUG: + disable_inds = bg_inds[:(len(bg_inds) - num_bg)] + labels[disable_inds] = -1 + + #fg_inds = np.where(labels == 1)[0] + #num_fg = len(fg_inds) + #num_bg = num_fg*int(1.0/config.TRAIN.RPN_FG_FRACTION-1) + + #bg_inds = np.where(labels == 0)[0] + #if len(bg_inds) > num_bg: + # disable_inds = npr.choice(bg_inds, size=(len(bg_inds) - num_bg), replace=False) + # if DEBUG: + # disable_inds = bg_inds[:(len(bg_inds) - num_bg)] + # labels[disable_inds] = -1 + else: + fg_inds = np.where(labels == 1)[0] + num_fg = len(fg_inds) + bg_inds = np.where(labels == 0)[0] + num_bg = len(bg_inds) + + #print('anchor stat', num_fg, num_bg) + + bbox_targets = np.zeros((len(inds_inside), bbox_pred_len), + dtype=np.float32) + if gt_boxes.size > 0: + #print('GT', gt_boxes.shape, gt_boxes[argmax_overlaps, :4].shape) + bbox_targets[:, :] = bbox_transform(anchors, + gt_boxes[argmax_overlaps, :]) + #bbox_targets[:,4] = gt_blur + #tb = datetime.datetime.now() + #self._times[1] += (tb-ta).total_seconds() + #ta = datetime.datetime.now() + + bbox_weights = np.zeros((len(inds_inside), bbox_pred_len), + dtype=np.float32) + #bbox_weights[labels == 1, :] = np.array(config.TRAIN.RPN_BBOX_WEIGHTS) + bbox_weights[labels == 1, 0:4] = 1.0 + if bbox_pred_len > 4: + bbox_weights[labels == 1, 4:bbox_pred_len] = 0.1 + + if landmark: + landmark_targets = np.zeros((len(inds_inside), landmark_pred_len), + dtype=np.float32) + #landmark_weights = np.zeros((len(inds_inside), 10), dtype=np.float32) + landmark_weights = np.zeros((len(inds_inside), landmark_pred_len), + dtype=np.float32) + #landmark_weights[labels == 1, :] = np.array(config.TRAIN.RPN_LANDMARK_WEIGHTS) + if landmark_pred_len == 10: + landmark_weights[labels == 1, :] = 1.0 + elif landmark_pred_len == 15: + v = [1.0, 1.0, 0.1] * 5 + assert len(v) == 15 + landmark_weights[labels == 1, :] = np.array(v) + else: + assert False + #TODO here + if gt_landmarks.size > 0: + #print('AAA',argmax_overlaps) + a_landmarks = gt_landmarks[argmax_overlaps, :, :] + landmark_targets[:] = landmark_transform(anchors, a_landmarks) + invalid = np.where(a_landmarks[:, 0, 2] < 0.0)[0] + #assert len(invalid)==0 + #landmark_weights[invalid, :] = np.array(config.TRAIN.RPN_INVALID_LANDMARK_WEIGHTS) + landmark_weights[invalid, :] = 0.0 + #tb = datetime.datetime.now() + #self._times[2] += (tb-ta).total_seconds() + #ta = datetime.datetime.now() + + #if DEBUG: + # _sums = bbox_targets[labels == 1, :].sum(axis=0) + # _squared_sums = (bbox_targets[labels == 1, :] ** 2).sum(axis=0) + # _counts = np.sum(labels == 1) + # means = _sums / (_counts + 1e-14) + # stds = np.sqrt(_squared_sums / _counts - means ** 2) + # print 'means', means + # print 'stdevs', stds + # map up to original set of anchors + #print(labels.shape, total_anchors, inds_inside.shape, inds_inside[0], inds_inside[-1]) + labels = AA._unmap(labels, total_anchors, inds_inside, fill=-1) + bbox_targets = AA._unmap(bbox_targets, + total_anchors, + inds_inside, + fill=0) + bbox_weights = AA._unmap(bbox_weights, + total_anchors, + inds_inside, + fill=0) + if landmark: + landmark_targets = AA._unmap(landmark_targets, + total_anchors, + inds_inside, + fill=0) + landmark_weights = AA._unmap(landmark_weights, + total_anchors, + inds_inside, + fill=0) + #print('CC', anchors.shape, len(inds_inside)) + + bbox_targets[:, + 0::4] = bbox_targets[:, 0::4] / config.TRAIN.BBOX_STDS[0] + bbox_targets[:, + 1::4] = bbox_targets[:, 1::4] / config.TRAIN.BBOX_STDS[1] + bbox_targets[:, + 2::4] = bbox_targets[:, 2::4] / config.TRAIN.BBOX_STDS[2] + bbox_targets[:, + 3::4] = bbox_targets[:, 3::4] / config.TRAIN.BBOX_STDS[3] + landmark_targets /= config.TRAIN.LANDMARK_STD + #print('applied STD') + + #if DEBUG: + # if gt_boxes.size > 0: + # print 'rpn: max max_overlaps', np.max(max_overlaps) + # print 'rpn: num_positives', np.sum(labels == 1) + # print 'rpn: num_negatives', np.sum(labels == 0) + # _fg_sum = np.sum(labels == 1) + # _bg_sum = np.sum(labels == 0) + # _count = 1 + # print 'rpn: num_positive avg', _fg_sum / _count + # print 'rpn: num_negative avg', _bg_sum / _count + + # resahpe + label_list = list() + bbox_target_list = list() + bbox_weight_list = list() + if landmark: + landmark_target_list = list() + landmark_weight_list = list() + anchors_num_range = [0] + anchors_num_list + label = {} + for i in range(len(feat_strides)): + stride = feat_strides[i] + feat_height, feat_width = feat_infos[i] + A = A_list[i] + _label = labels[sum(anchors_num_range[:i + 1] + ):sum(anchors_num_range[:i + 1]) + + anchors_num_range[i + 1]] + if select_stride > 0 and stride != select_stride: + #print('set', stride, select_stride) + _label[:] = -1 + #print('_label', _label.shape, select_stride) + #_fg_inds = np.where(_label == 1)[0] + #n_fg = len(_fg_inds) + #STAT[0]+=1 + #STAT[stride]+=n_fg + #if STAT[0]%100==0: + # print('rpn_stat', STAT, file=sys.stderr) + bbox_target = bbox_targets[sum(anchors_num_range[:i + 1] + ):sum(anchors_num_range[:i + 1]) + + anchors_num_range[i + 1]] + bbox_weight = bbox_weights[sum(anchors_num_range[:i + 1] + ):sum(anchors_num_range[:i + 1]) + + anchors_num_range[i + 1]] + if landmark: + landmark_target = landmark_targets[ + sum(anchors_num_range[:i + + 1]):sum(anchors_num_range[:i + 1]) + + anchors_num_range[i + 1]] + landmark_weight = landmark_weights[ + sum(anchors_num_range[:i + + 1]):sum(anchors_num_range[:i + 1]) + + anchors_num_range[i + 1]] + + _label = _label.reshape( + (1, feat_height, feat_width, A)).transpose(0, 3, 1, 2) + _label = _label.reshape((1, A * feat_height * feat_width)) + bbox_target = bbox_target.reshape( + (1, feat_height * feat_width, + A * bbox_pred_len)).transpose(0, 2, 1) + bbox_weight = bbox_weight.reshape( + (1, feat_height * feat_width, A * bbox_pred_len)).transpose( + (0, 2, 1)) + label['%s_label_stride%d' % (prefix, stride)] = _label + label['%s_bbox_target_stride%d' % (prefix, stride)] = bbox_target + label['%s_bbox_weight_stride%d' % (prefix, stride)] = bbox_weight + if landmark: + landmark_target = landmark_target.reshape( + (1, feat_height * feat_width, + A * landmark_pred_len)).transpose(0, 2, 1) + landmark_weight = landmark_weight.reshape( + (1, feat_height * feat_width, + A * landmark_pred_len)).transpose((0, 2, 1)) + label['%s_landmark_target_stride%d' % + (prefix, stride)] = landmark_target + label['%s_landmark_weight_stride%d' % + (prefix, stride)] = landmark_weight + #print('in_rpn', stride,_label.shape, bbox_target.shape, bbox_weight.shape, file=sys.stderr) + label_list.append(_label) + #print('DD', _label.shape) + bbox_target_list.append(bbox_target) + bbox_weight_list.append(bbox_weight) + if landmark: + landmark_target_list.append(landmark_target) + landmark_weight_list.append(landmark_weight) + + label_concat = np.concatenate(label_list, axis=1) + bbox_target_concat = np.concatenate(bbox_target_list, axis=2) + bbox_weight_concat = np.concatenate(bbox_weight_list, axis=2) + #fg_inds = np.where(label_concat[0] == 1)[0] + #print('fg_inds_in_rpn2', fg_inds, file=sys.stderr) + + label.update({ + '%s_label' % prefix: label_concat, + '%s_bbox_target' % prefix: bbox_target_concat, + '%s_bbox_weight' % prefix: bbox_weight_concat + }) + if landmark: + landmark_target_concat = np.concatenate(landmark_target_list, + axis=2) + landmark_weight_concat = np.concatenate(landmark_weight_list, + axis=2) + label['%s_landmark_target' % prefix] = landmark_target_concat + label['%s_landmark_weight' % prefix] = landmark_weight_concat + #tb = datetime.datetime.now() + #self._times[3] += (tb-ta).total_seconds() + #ta = datetime.datetime.now() + #print(self._times) + return label diff --git a/RetinaFace/rcnn/logger.py b/detection/RetinaFace/rcnn/logger.py similarity index 100% rename from RetinaFace/rcnn/logger.py rename to detection/RetinaFace/rcnn/logger.py diff --git a/RetinaFace/rcnn/processing/__init__.py b/detection/RetinaFace/rcnn/processing/__init__.py similarity index 100% rename from RetinaFace/rcnn/processing/__init__.py rename to detection/RetinaFace/rcnn/processing/__init__.py diff --git a/RetinaFaceAntiCov/rcnn/processing/assign_levels.py b/detection/RetinaFace/rcnn/processing/assign_levels.py similarity index 73% rename from RetinaFaceAntiCov/rcnn/processing/assign_levels.py rename to detection/RetinaFace/rcnn/processing/assign_levels.py index b237439..012d73d 100755 --- a/RetinaFaceAntiCov/rcnn/processing/assign_levels.py +++ b/detection/RetinaFace/rcnn/processing/assign_levels.py @@ -3,7 +3,8 @@ import numpy as np def compute_assign_targets(rois, threshold): - rois_area = np.sqrt((rois[:, 2] - rois[:, 0] + 1) * (rois[:, 3] - rois[:, 1] + 1)) + rois_area = np.sqrt( + (rois[:, 2] - rois[:, 0] + 1) * (rois[:, 3] - rois[:, 1] + 1)) num_rois = np.shape(rois)[0] assign_levels = np.zeros(num_rois, dtype=np.uint8) for i, stride in enumerate(config.RCNN_FEAT_STRIDE): @@ -24,14 +25,12 @@ def add_assign_targets(roidb): assert len(roidb) > 0 assert 'boxes' in roidb[0] - area_threshold = [[np.inf, 448], - [448, 224], - [224, 112], - [112, 0]] + area_threshold = [[np.inf, 448], [448, 224], [224, 112], [112, 0]] assert len(config.RCNN_FEAT_STRIDE) == len(area_threshold) num_images = len(roidb) for im_i in range(num_images): rois = roidb[im_i]['boxes'] - roidb[im_i]['assign_levels'] = compute_assign_targets(rois, area_threshold) + roidb[im_i]['assign_levels'] = compute_assign_targets( + rois, area_threshold) diff --git a/RetinaFaceAntiCov/rcnn/processing/bbox_regression.py b/detection/RetinaFace/rcnn/processing/bbox_regression.py similarity index 93% rename from RetinaFaceAntiCov/rcnn/processing/bbox_regression.py rename to detection/RetinaFace/rcnn/processing/bbox_regression.py index 11d1a02..0eaf917 100644 --- a/RetinaFaceAntiCov/rcnn/processing/bbox_regression.py +++ b/detection/RetinaFace/rcnn/processing/bbox_regression.py @@ -69,7 +69,8 @@ def add_bbox_regression_targets(roidb): rois = roidb[im_i]['boxes'] max_overlaps = roidb[im_i]['max_overlaps'] max_classes = roidb[im_i]['max_classes'] - roidb[im_i]['bbox_targets'] = compute_bbox_regression_targets(rois, max_overlaps, max_classes) + roidb[im_i]['bbox_targets'] = compute_bbox_regression_targets( + rois, max_overlaps, max_classes) if config.TRAIN.BBOX_NORMALIZATION_PRECOMPUTED: # use fixed / precomputed means and stds instead of empirical values @@ -87,11 +88,12 @@ def add_bbox_regression_targets(roidb): if cls_indexes.size > 0: class_counts[cls] += cls_indexes.size sums[cls, :] += targets[cls_indexes, 1:].sum(axis=0) - squared_sums[cls, :] += (targets[cls_indexes, 1:] ** 2).sum(axis=0) + squared_sums[cls, :] += (targets[cls_indexes, + 1:]**2).sum(axis=0) means = sums / class_counts # var(x) = E(x^2) - E(x)^2 - stds = np.sqrt(squared_sums / class_counts - means ** 2) + stds = np.sqrt(squared_sums / class_counts - means**2) # normalized targets for im_i in range(num_images): @@ -142,7 +144,8 @@ def compute_mask_and_label(ex_rois, ex_labels, seg, flipped): mask_target = np.zeros((n_rois, 28, 28), dtype=np.int8) mask_label = np.zeros((n_rois), dtype=np.int8) for n in range(n_rois): - target = ins_seg[int(rois[n, 1]): int(rois[n, 3]), int(rois[n, 0]): int(rois[n, 2])] + target = ins_seg[int(rois[n, 1]):int(rois[n, 3]), + int(rois[n, 0]):int(rois[n, 2])] ids = np.unique(target) ins_id = 0 max_count = 0 @@ -158,8 +161,9 @@ def compute_mask_and_label(ex_rois, ex_labels, seg, flipped): x2 = min(rois[n, 2], x_max) y2 = min(rois[n, 3], y_max) iou = (x2 - x1) * (y2 - y1) - iou = iou / ((rois[n, 2] - rois[n, 0]) * (rois[n, 3] - rois[n, 1]) - + (x_max - x_min) * (y_max - y_min) - iou) + iou = iou / ((rois[n, 2] - rois[n, 0]) * + (rois[n, 3] - rois[n, 1]) + (x_max - x_min) * + (y_max - y_min) - iou) if iou > max_count: ins_id = id max_count = iou @@ -202,16 +206,17 @@ def compute_bbox_mask_targets_and_label(rois, overlaps, labels, seg, flipped): # Get IoU overlap between each ex ROI and gt ROI ex_gt_overlaps = bbox_overlaps(rois[ex_inds, :], rois[gt_inds, :]) - # Find which gt ROI each ex ROI has max overlap with: # this will be the ex ROI's gt target gt_assignment = ex_gt_overlaps.argmax(axis=1) gt_rois = rois[gt_inds[gt_assignment], :] ex_rois = rois[ex_inds, :] - mask_targets, mask_label = compute_mask_and_label(ex_rois, labels[ex_inds], seg, flipped) + mask_targets, mask_label = compute_mask_and_label(ex_rois, labels[ex_inds], + seg, flipped) return mask_targets, mask_label, ex_inds + def add_mask_targets(roidb): """ given roidb, add ['bbox_targets'] and normalize bounding box regression targets @@ -240,9 +245,12 @@ def add_mask_targets(roidb): flipped = roidb[im_i]['flipped'] roidb[im_i]['mask_targets'], roidb[im_i]['mask_labels'], roidb[im_i]['mask_inds'] = \ compute_bbox_mask_targets_and_label(rois, max_overlaps, max_classes, ins_seg, flipped) + threads = [threading.Thread(target=process, args=()) for i in range(10)] - for t in threads: t.start() - for t in threads: t.join() + for t in threads: + t.start() + for t in threads: + t.join() # Single thread # for im_i in range(num_images): # print "-----processing img {}".format(im_i) diff --git a/RetinaFace/rcnn/processing/bbox_transform.py b/detection/RetinaFace/rcnn/processing/bbox_transform.py similarity index 79% rename from RetinaFace/rcnn/processing/bbox_transform.py rename to detection/RetinaFace/rcnn/processing/bbox_transform.py index 0c4fc7c..5ee3646 100644 --- a/RetinaFace/rcnn/processing/bbox_transform.py +++ b/detection/RetinaFace/rcnn/processing/bbox_transform.py @@ -18,13 +18,17 @@ def bbox_overlaps_py(boxes, query_boxes): k_ = query_boxes.shape[0] overlaps = np.zeros((n_, k_), dtype=np.float) for k in range(k_): - query_box_area = (query_boxes[k, 2] - query_boxes[k, 0] + 1) * (query_boxes[k, 3] - query_boxes[k, 1] + 1) + query_box_area = (query_boxes[k, 2] - query_boxes[k, 0] + + 1) * (query_boxes[k, 3] - query_boxes[k, 1] + 1) for n in range(n_): - iw = min(boxes[n, 2], query_boxes[k, 2]) - max(boxes[n, 0], query_boxes[k, 0]) + 1 + iw = min(boxes[n, 2], query_boxes[k, 2]) - max( + boxes[n, 0], query_boxes[k, 0]) + 1 if iw > 0: - ih = min(boxes[n, 3], query_boxes[k, 3]) - max(boxes[n, 1], query_boxes[k, 1]) + 1 + ih = min(boxes[n, 3], query_boxes[k, 3]) - max( + boxes[n, 1], query_boxes[k, 1]) + 1 if ih > 0: - box_area = (boxes[n, 2] - boxes[n, 0] + 1) * (boxes[n, 3] - boxes[n, 1] + 1) + box_area = (boxes[n, 2] - boxes[n, 0] + + 1) * (boxes[n, 3] - boxes[n, 1] + 1) all_area = float(box_area + query_box_area - iw * ih) overlaps[n, k] = iw * ih / all_area return overlaps @@ -72,18 +76,19 @@ def nonlinear_transform(ex_rois, gt_rois): targets_dw = np.log(gt_widths / ex_widths) targets_dh = np.log(gt_heights / ex_heights) - if gt_rois.shape[1]<=4: - targets = np.vstack( - (targets_dx, targets_dy, targets_dw, targets_dh)).transpose() - return targets + if gt_rois.shape[1] <= 4: + targets = np.vstack( + (targets_dx, targets_dy, targets_dw, targets_dh)).transpose() + return targets else: - targets = [targets_dx, targets_dy, targets_dw, targets_dh] - #if config.USE_BLUR: - # for i in range(4, gt_rois.shape[1]): - # t = gt_rois[:,i] - # targets.append(t) - targets = np.vstack(targets).transpose() - return targets + targets = [targets_dx, targets_dy, targets_dw, targets_dh] + #if config.USE_BLUR: + # for i in range(4, gt_rois.shape[1]): + # t = gt_rois[:,i] + # targets.append(t) + targets = np.vstack(targets).transpose() + return targets + def landmark_transform(ex_rois, gt_rois): @@ -94,22 +99,20 @@ def landmark_transform(ex_rois, gt_rois): ex_ctr_x = ex_rois[:, 0] + 0.5 * (ex_widths - 1.0) ex_ctr_y = ex_rois[:, 1] + 0.5 * (ex_heights - 1.0) - targets = [] for i in range(gt_rois.shape[1]): - for j in range(gt_rois.shape[2]): - #if not config.USE_OCCLUSION and j==2: - # continue - if j==2: - continue - if j==0: #w - target = (gt_rois[:,i,j] - ex_ctr_x) / (ex_widths + 1e-14) - elif j==1: #h - target = (gt_rois[:,i,j] - ex_ctr_y) / (ex_heights + 1e-14) - else: #visibile - target = gt_rois[:,i,j] - targets.append(target) - + for j in range(gt_rois.shape[2]): + #if not config.USE_OCCLUSION and j==2: + # continue + if j == 2: + continue + if j == 0: #w + target = (gt_rois[:, i, j] - ex_ctr_x) / (ex_widths + 1e-14) + elif j == 1: #h + target = (gt_rois[:, i, j] - ex_ctr_y) / (ex_heights + 1e-14) + else: #visibile + target = gt_rois[:, i, j] + targets.append(target) targets = np.vstack(targets).transpose() return targets @@ -154,6 +157,7 @@ def nonlinear_pred(boxes, box_deltas): return pred_boxes + def landmark_pred(boxes, landmark_deltas): if boxes.shape[0] == 0: return np.zeros((0, landmark_deltas.shape[1])) @@ -164,14 +168,15 @@ def landmark_pred(boxes, landmark_deltas): ctr_y = boxes[:, 1] + 0.5 * (heights - 1.0) preds = [] for i in range(landmark_deltas.shape[1]): - if i%2==0: - pred = (landmark_deltas[:,i]*widths + ctr_x) - else: - pred = (landmark_deltas[:,i]*heights + ctr_y) - preds.append(pred) + if i % 2 == 0: + pred = (landmark_deltas[:, i] * widths + ctr_x) + else: + pred = (landmark_deltas[:, i] * heights + ctr_y) + preds.append(pred) preds = np.vstack(preds).transpose() return preds + def iou_transform(ex_rois, gt_rois): """ return bbox targets, IoU loss uses gt_rois as gt """ assert ex_rois.shape[0] == gt_rois.shape[0], 'inconsistent rois number' diff --git a/RetinaFaceAntiCov/rcnn/processing/generate_anchor.py b/detection/RetinaFace/rcnn/processing/generate_anchor.py similarity index 70% rename from RetinaFaceAntiCov/rcnn/processing/generate_anchor.py rename to detection/RetinaFace/rcnn/processing/generate_anchor.py index eb47d8e..83c5ada 100644 --- a/RetinaFaceAntiCov/rcnn/processing/generate_anchor.py +++ b/detection/RetinaFace/rcnn/processing/generate_anchor.py @@ -12,8 +12,12 @@ from ..cython.anchors import anchors_cython def anchors_plane(feat_h, feat_w, stride, base_anchor): return anchors_cython(feat_h, feat_w, stride, base_anchor) -def generate_anchors(base_size=16, ratios=[0.5, 1, 2], - scales=2 ** np.arange(3, 6), stride=16, dense_anchor=False): + +def generate_anchors(base_size=16, + ratios=[0.5, 1, 2], + scales=2**np.arange(3, 6), + stride=16, + dense_anchor=False): """ Generate anchor (reference) windows by enumerating aspect ratios X scales wrt a reference (0, 0, 15, 15) window. @@ -21,16 +25,19 @@ def generate_anchors(base_size=16, ratios=[0.5, 1, 2], base_anchor = np.array([1, 1, base_size, base_size]) - 1 ratio_anchors = _ratio_enum(base_anchor, ratios) - anchors = np.vstack([_scale_enum(ratio_anchors[i, :], scales) - for i in range(ratio_anchors.shape[0])]) + anchors = np.vstack([ + _scale_enum(ratio_anchors[i, :], scales) + for i in range(ratio_anchors.shape[0]) + ]) if dense_anchor: - assert stride%2==0 - anchors2 = anchors.copy() - anchors2[:,:] += int(stride/2) - anchors = np.vstack( (anchors, anchors2) ) + assert stride % 2 == 0 + anchors2 = anchors.copy() + anchors2[:, :] += int(stride / 2) + anchors = np.vstack((anchors, anchors2)) #print('GA',base_anchor.shape, ratio_anchors.shape, anchors.shape) return anchors + #def generate_anchors_fpn(base_size=[64,32,16,8,4], ratios=[0.5, 1, 2], scales=8): # """ # Generate anchor (reference) windows by enumerating aspect ratios X @@ -48,33 +55,35 @@ def generate_anchors(base_size=16, ratios=[0.5, 1, 2], # anchors.append(r) # return anchors -def generate_anchors_fpn(dense_anchor=False, cfg = None): + +def generate_anchors_fpn(dense_anchor=False, cfg=None): #assert(False) """ Generate anchor (reference) windows by enumerating aspect ratios X scales wrt a reference (0, 0, 15, 15) window. """ if cfg is None: - from ..config import config - cfg = config.RPN_ANCHOR_CFG + from ..config import config + cfg = config.RPN_ANCHOR_CFG RPN_FEAT_STRIDE = [] for k in cfg: - RPN_FEAT_STRIDE.append( int(k) ) + RPN_FEAT_STRIDE.append(int(k)) RPN_FEAT_STRIDE = sorted(RPN_FEAT_STRIDE, reverse=True) anchors = [] for k in RPN_FEAT_STRIDE: - v = cfg[str(k)] - bs = v['BASE_SIZE'] - __ratios = np.array(v['RATIOS']) - __scales = np.array(v['SCALES']) - stride = int(k) - #print('anchors_fpn', bs, __ratios, __scales, file=sys.stderr) - r = generate_anchors(bs, __ratios, __scales, stride, dense_anchor) - #print('anchors_fpn', r.shape, file=sys.stderr) - anchors.append(r) + v = cfg[str(k)] + bs = v['BASE_SIZE'] + __ratios = np.array(v['RATIOS']) + __scales = np.array(v['SCALES']) + stride = int(k) + #print('anchors_fpn', bs, __ratios, __scales, file=sys.stderr) + r = generate_anchors(bs, __ratios, __scales, stride, dense_anchor) + #print('anchors_fpn', r.shape, file=sys.stderr) + anchors.append(r) return anchors + def _whctrs(anchor): """ Return width, height, x center, and y center for an anchor (window). @@ -95,10 +104,8 @@ def _mkanchors(ws, hs, x_ctr, y_ctr): ws = ws[:, np.newaxis] hs = hs[:, np.newaxis] - anchors = np.hstack((x_ctr - 0.5 * (ws - 1), - y_ctr - 0.5 * (hs - 1), - x_ctr + 0.5 * (ws - 1), - y_ctr + 0.5 * (hs - 1))) + anchors = np.hstack((x_ctr - 0.5 * (ws - 1), y_ctr - 0.5 * (hs - 1), + x_ctr + 0.5 * (ws - 1), y_ctr + 0.5 * (hs - 1))) return anchors diff --git a/RetinaFace/rcnn/processing/nms.py b/detection/RetinaFace/rcnn/processing/nms.py similarity index 99% rename from RetinaFace/rcnn/processing/nms.py rename to detection/RetinaFace/rcnn/processing/nms.py index 230139c..b32d92d 100644 --- a/RetinaFace/rcnn/processing/nms.py +++ b/detection/RetinaFace/rcnn/processing/nms.py @@ -9,18 +9,21 @@ except ImportError: def py_nms_wrapper(thresh): def _nms(dets): return nms(dets, thresh) + return _nms def cpu_nms_wrapper(thresh): def _nms(dets): return cpu_nms(dets, thresh) + return _nms def gpu_nms_wrapper(thresh, device_id): def _nms(dets): return gpu_nms(dets, thresh, device_id) + if gpu_nms is not None: return _nms else: diff --git a/RetinaFace/rcnn/pycocotools/UPSTREAM_REV b/detection/RetinaFace/rcnn/pycocotools/UPSTREAM_REV similarity index 100% rename from RetinaFace/rcnn/pycocotools/UPSTREAM_REV rename to detection/RetinaFace/rcnn/pycocotools/UPSTREAM_REV diff --git a/RetinaFace/rcnn/pycocotools/__init__.py b/detection/RetinaFace/rcnn/pycocotools/__init__.py similarity index 100% rename from RetinaFace/rcnn/pycocotools/__init__.py rename to detection/RetinaFace/rcnn/pycocotools/__init__.py diff --git a/RetinaFace/rcnn/pycocotools/_mask.c b/detection/RetinaFace/rcnn/pycocotools/_mask.c similarity index 100% rename from RetinaFace/rcnn/pycocotools/_mask.c rename to detection/RetinaFace/rcnn/pycocotools/_mask.c diff --git a/RetinaFace/rcnn/pycocotools/_mask.pyx b/detection/RetinaFace/rcnn/pycocotools/_mask.pyx similarity index 100% rename from RetinaFace/rcnn/pycocotools/_mask.pyx rename to detection/RetinaFace/rcnn/pycocotools/_mask.pyx diff --git a/RetinaFace/rcnn/pycocotools/coco.py b/detection/RetinaFace/rcnn/pycocotools/coco.py similarity index 77% rename from RetinaFace/rcnn/pycocotools/coco.py rename to detection/RetinaFace/rcnn/pycocotools/coco.py index 2b63ea1..4f79236 100644 --- a/RetinaFace/rcnn/pycocotools/coco.py +++ b/detection/RetinaFace/rcnn/pycocotools/coco.py @@ -62,6 +62,7 @@ if PYTHON_VERSION == 2: elif PYTHON_VERSION == 3: from urllib.request import urlretrieve + class COCO: def __init__(self, annotation_file=None): """ @@ -71,14 +72,18 @@ class COCO: :return: """ # load dataset - self.dataset,self.anns,self.cats,self.imgs = dict(),dict(),dict(),dict() + self.dataset, self.anns, self.cats, self.imgs = dict(), dict(), dict( + ), dict() self.imgToAnns, self.catToImgs = defaultdict(list), defaultdict(list) if not annotation_file == None: print('loading annotations into memory...') tic = time.time() dataset = json.load(open(annotation_file, 'r')) - assert type(dataset)==dict, 'annotation file format {} not supported'.format(type(dataset)) - print('Done (t={:0.2f}s)'.format(time.time()- tic)) + assert type( + dataset + ) == dict, 'annotation file format {} not supported'.format( + type(dataset)) + print('Done (t={:0.2f}s)'.format(time.time() - tic)) self.dataset = dataset self.createIndex() @@ -86,7 +91,7 @@ class COCO: # create index print('creating index...') anns, cats, imgs = {}, {}, {} - imgToAnns,catToImgs = defaultdict(list),defaultdict(list) + imgToAnns, catToImgs = defaultdict(list), defaultdict(list) if 'annotations' in self.dataset: for ann in self.dataset['annotations']: imgToAnns[ann['image_id']].append(ann) @@ -137,12 +142,20 @@ class COCO: anns = self.dataset['annotations'] else: if not len(imgIds) == 0: - lists = [self.imgToAnns[imgId] for imgId in imgIds if imgId in self.imgToAnns] + lists = [ + self.imgToAnns[imgId] for imgId in imgIds + if imgId in self.imgToAnns + ] anns = list(itertools.chain.from_iterable(lists)) else: anns = self.dataset['annotations'] - anns = anns if len(catIds) == 0 else [ann for ann in anns if ann['category_id'] in catIds] - anns = anns if len(areaRng) == 0 else [ann for ann in anns if ann['area'] > areaRng[0] and ann['area'] < areaRng[1]] + anns = anns if len(catIds) == 0 else [ + ann for ann in anns if ann['category_id'] in catIds + ] + anns = anns if len(areaRng) == 0 else [ + ann for ann in anns + if ann['area'] > areaRng[0] and ann['area'] < areaRng[1] + ] if not iscrowd == None: ids = [ann['id'] for ann in anns if ann['iscrowd'] == iscrowd] else: @@ -165,9 +178,15 @@ class COCO: cats = self.dataset['categories'] else: cats = self.dataset['categories'] - cats = cats if len(catNms) == 0 else [cat for cat in cats if cat['name'] in catNms] - cats = cats if len(supNms) == 0 else [cat for cat in cats if cat['supercategory'] in supNms] - cats = cats if len(catIds) == 0 else [cat for cat in cats if cat['id'] in catIds] + cats = cats if len(catNms) == 0 else [ + cat for cat in cats if cat['name'] in catNms + ] + cats = cats if len(supNms) == 0 else [ + cat for cat in cats if cat['supercategory'] in supNms + ] + cats = cats if len(catIds) == 0 else [ + cat for cat in cats if cat['id'] in catIds + ] ids = [cat['id'] for cat in cats] return ids @@ -245,45 +264,67 @@ class COCO: polygons = [] color = [] for ann in anns: - c = (np.random.random((1, 3))*0.6+0.4).tolist()[0] + c = (np.random.random((1, 3)) * 0.6 + 0.4).tolist()[0] if 'segmentation' in ann: if type(ann['segmentation']) == list: # polygon for seg in ann['segmentation']: - poly = np.array(seg).reshape((int(len(seg)/2), 2)) + poly = np.array(seg).reshape( + (int(len(seg) / 2), 2)) polygons.append(Polygon(poly)) color.append(c) else: # mask t = self.imgs[ann['image_id']] if type(ann['segmentation']['counts']) == list: - rle = maskUtils.frPyObjects([ann['segmentation']], t['height'], t['width']) + rle = maskUtils.frPyObjects([ann['segmentation']], + t['height'], + t['width']) else: rle = [ann['segmentation']] m = maskUtils.decode(rle) - img = np.ones( (m.shape[0], m.shape[1], 3) ) + img = np.ones((m.shape[0], m.shape[1], 3)) if ann['iscrowd'] == 1: - color_mask = np.array([2.0,166.0,101.0])/255 + color_mask = np.array([2.0, 166.0, 101.0]) / 255 if ann['iscrowd'] == 0: color_mask = np.random.random((1, 3)).tolist()[0] for i in range(3): - img[:,:,i] = color_mask[i] - ax.imshow(np.dstack( (img, m*0.5) )) + img[:, :, i] = color_mask[i] + ax.imshow(np.dstack((img, m * 0.5))) if 'keypoints' in ann and type(ann['keypoints']) == list: # turn skeleton into zero-based index - sks = np.array(self.loadCats(ann['category_id'])[0]['skeleton'])-1 + sks = np.array( + self.loadCats(ann['category_id'])[0]['skeleton']) - 1 kp = np.array(ann['keypoints']) x = kp[0::3] y = kp[1::3] v = kp[2::3] for sk in sks: - if np.all(v[sk]>0): - plt.plot(x[sk],y[sk], linewidth=3, color=c) - plt.plot(x[v>0], y[v>0],'o',markersize=8, markerfacecolor=c, markeredgecolor='k',markeredgewidth=2) - plt.plot(x[v>1], y[v>1],'o',markersize=8, markerfacecolor=c, markeredgecolor=c, markeredgewidth=2) - p = PatchCollection(polygons, facecolor=color, linewidths=0, alpha=0.4) + if np.all(v[sk] > 0): + plt.plot(x[sk], y[sk], linewidth=3, color=c) + plt.plot(x[v > 0], + y[v > 0], + 'o', + markersize=8, + markerfacecolor=c, + markeredgecolor='k', + markeredgewidth=2) + plt.plot(x[v > 1], + y[v > 1], + 'o', + markersize=8, + markerfacecolor=c, + markeredgecolor=c, + markeredgewidth=2) + p = PatchCollection(polygons, + facecolor=color, + linewidths=0, + alpha=0.4) ax.add_collection(p) - p = PatchCollection(polygons, facecolor='none', edgecolors=color, linewidths=2) + p = PatchCollection(polygons, + facecolor='none', + edgecolors=color, + linewidths=2) ax.add_collection(p) elif datasetType == 'captions': for ann in anns: @@ -311,46 +352,52 @@ class COCO: assert set(annsImgIds) == (set(annsImgIds) & set(self.getImgIds())), \ 'Results do not correspond to current coco set' if 'caption' in anns[0]: - imgIds = set([img['id'] for img in res.dataset['images']]) & set([ann['image_id'] for ann in anns]) - res.dataset['images'] = [img for img in res.dataset['images'] if img['id'] in imgIds] + imgIds = set([img['id'] for img in res.dataset['images']]) & set( + [ann['image_id'] for ann in anns]) + res.dataset['images'] = [ + img for img in res.dataset['images'] if img['id'] in imgIds + ] for id, ann in enumerate(anns): - ann['id'] = id+1 + ann['id'] = id + 1 elif 'bbox' in anns[0] and not anns[0]['bbox'] == []: - res.dataset['categories'] = copy.deepcopy(self.dataset['categories']) + res.dataset['categories'] = copy.deepcopy( + self.dataset['categories']) for id, ann in enumerate(anns): bb = ann['bbox'] - x1, x2, y1, y2 = [bb[0], bb[0]+bb[2], bb[1], bb[1]+bb[3]] + x1, x2, y1, y2 = [bb[0], bb[0] + bb[2], bb[1], bb[1] + bb[3]] if not 'segmentation' in ann: ann['segmentation'] = [[x1, y1, x1, y2, x2, y2, x2, y1]] - ann['area'] = bb[2]*bb[3] - ann['id'] = id+1 + ann['area'] = bb[2] * bb[3] + ann['id'] = id + 1 ann['iscrowd'] = 0 elif 'segmentation' in anns[0]: - res.dataset['categories'] = copy.deepcopy(self.dataset['categories']) + res.dataset['categories'] = copy.deepcopy( + self.dataset['categories']) for id, ann in enumerate(anns): # now only support compressed RLE format as segmentation results ann['area'] = maskUtils.area(ann['segmentation']) if not 'bbox' in ann: ann['bbox'] = maskUtils.toBbox(ann['segmentation']) - ann['id'] = id+1 + ann['id'] = id + 1 ann['iscrowd'] = 0 elif 'keypoints' in anns[0]: - res.dataset['categories'] = copy.deepcopy(self.dataset['categories']) + res.dataset['categories'] = copy.deepcopy( + self.dataset['categories']) for id, ann in enumerate(anns): s = ann['keypoints'] x = s[0::3] y = s[1::3] - x0,x1,y0,y1 = np.min(x), np.max(x), np.min(y), np.max(y) - ann['area'] = (x1-x0)*(y1-y0) + x0, x1, y0, y1 = np.min(x), np.max(x), np.min(y), np.max(y) + ann['area'] = (x1 - x0) * (y1 - y0) ann['id'] = id + 1 - ann['bbox'] = [x0,y0,x1-x0,y1-y0] - print('DONE (t={:0.2f}s)'.format(time.time()- tic)) + ann['bbox'] = [x0, y0, x1 - x0, y1 - y0] + print('DONE (t={:0.2f}s)'.format(time.time() - tic)) res.dataset['annotations'] = anns res.createIndex() return res - def download(self, tarDir = None, imgIds = [] ): + def download(self, tarDir=None, imgIds=[]): ''' Download COCO images from mscoco.org server. :param tarDir (str): COCO results directory name @@ -372,7 +419,9 @@ class COCO: fname = os.path.join(tarDir, img['file_name']) if not os.path.exists(fname): urlretrieve(img['coco_url'], fname) - print('downloaded {}/{} images (t={:0.1f}s)'.format(i, N, time.time()- tic)) + print('downloaded {}/{} images (t={:0.1f}s)'.format( + i, N, + time.time() - tic)) def loadNumpyAnnotations(self, data): """ @@ -381,20 +430,20 @@ class COCO: :return: annotations (python nested list) """ print('Converting ndarray to lists...') - assert(type(data) == np.ndarray) + assert (type(data) == np.ndarray) print(data.shape) - assert(data.shape[1] == 7) + assert (data.shape[1] == 7) N = data.shape[0] ann = [] for i in range(N): if i % 1000000 == 0: - print('{}/{}'.format(i,N)) + print('{}/{}'.format(i, N)) ann += [{ - 'image_id' : int(data[i, 0]), - 'bbox' : [ data[i, 1], data[i, 2], data[i, 3], data[i, 4] ], - 'score' : data[i, 5], + 'image_id': int(data[i, 0]), + 'bbox': [data[i, 1], data[i, 2], data[i, 3], data[i, 4]], + 'score': data[i, 5], 'category_id': int(data[i, 6]), - }] + }] return ann def annToRLE(self, ann): diff --git a/RetinaFace/rcnn/pycocotools/cocoeval.py b/detection/RetinaFace/rcnn/pycocotools/cocoeval.py similarity index 64% rename from RetinaFace/rcnn/pycocotools/cocoeval.py rename to detection/RetinaFace/rcnn/pycocotools/cocoeval.py index 58975db..1d35318 100644 --- a/RetinaFace/rcnn/pycocotools/cocoeval.py +++ b/detection/RetinaFace/rcnn/pycocotools/cocoeval.py @@ -7,6 +7,7 @@ from collections import defaultdict from .mask import * import copy + class COCOeval: # Interface for evaluating detection on the Microsoft COCO dataset. # @@ -66,22 +67,22 @@ class COCOeval: ''' if not iouType: print('iouType not specified. use default iouType segm') - self.cocoGt = cocoGt # ground truth COCO API - self.cocoDt = cocoDt # detections COCO API - self.params = {} # evaluation parameters - self.evalImgs = defaultdict(list) # per-image per-category evaluation results [KxAxI] elements - self.eval = {} # accumulated evaluation results - self._gts = defaultdict(list) # gt for evaluation - self._dts = defaultdict(list) # dt for evaluation - self.params = Params(iouType=iouType) # parameters - self._paramsEval = {} # parameters for evaluation - self.stats = [] # result summarization - self.ious = {} # ious between all gts and dts + self.cocoGt = cocoGt # ground truth COCO API + self.cocoDt = cocoDt # detections COCO API + self.params = {} # evaluation parameters + self.evalImgs = defaultdict( + list) # per-image per-category evaluation results [KxAxI] elements + self.eval = {} # accumulated evaluation results + self._gts = defaultdict(list) # gt for evaluation + self._dts = defaultdict(list) # dt for evaluation + self.params = Params(iouType=iouType) # parameters + self._paramsEval = {} # parameters for evaluation + self.stats = [] # result summarization + self.ious = {} # ious between all gts and dts if not cocoGt is None: self.params.imgIds = sorted(cocoGt.getImgIds()) self.params.catIds = sorted(cocoGt.getCatIds()) - def _prepare(self): ''' Prepare ._gts and ._dts for evaluation based on params @@ -92,13 +93,16 @@ class COCOeval: for ann in anns: rle = coco.annToRLE(ann) ann['segmentation'] = rle + p = self.params if p.useCats: - gts=self.cocoGt.loadAnns(self.cocoGt.getAnnIds(imgIds=p.imgIds, catIds=p.catIds)) - dts=self.cocoDt.loadAnns(self.cocoDt.getAnnIds(imgIds=p.imgIds, catIds=p.catIds)) + gts = self.cocoGt.loadAnns( + self.cocoGt.getAnnIds(imgIds=p.imgIds, catIds=p.catIds)) + dts = self.cocoDt.loadAnns( + self.cocoDt.getAnnIds(imgIds=p.imgIds, catIds=p.catIds)) else: - gts=self.cocoGt.loadAnns(self.cocoGt.getAnnIds(imgIds=p.imgIds)) - dts=self.cocoDt.loadAnns(self.cocoDt.getAnnIds(imgIds=p.imgIds)) + gts = self.cocoGt.loadAnns(self.cocoGt.getAnnIds(imgIds=p.imgIds)) + dts = self.cocoDt.loadAnns(self.cocoDt.getAnnIds(imgIds=p.imgIds)) # convert ground truth to mask if iouType == 'segm' if p.iouType == 'segm': @@ -110,14 +114,15 @@ class COCOeval: gt['ignore'] = 'iscrowd' in gt and gt['iscrowd'] if p.iouType == 'keypoints': gt['ignore'] = (gt['num_keypoints'] == 0) or gt['ignore'] - self._gts = defaultdict(list) # gt for evaluation - self._dts = defaultdict(list) # dt for evaluation + self._gts = defaultdict(list) # gt for evaluation + self._dts = defaultdict(list) # dt for evaluation for gt in gts: self._gts[gt['image_id'], gt['category_id']].append(gt) for dt in dts: self._dts[dt['image_id'], dt['category_id']].append(dt) - self.evalImgs = defaultdict(list) # per-image per-category evaluation results - self.eval = {} # accumulated evaluation results + self.evalImgs = defaultdict( + list) # per-image per-category evaluation results + self.eval = {} # accumulated evaluation results def evaluate(self): ''' @@ -130,13 +135,14 @@ class COCOeval: # add backward compatibility if useSegm is specified in params if not p.useSegm is None: p.iouType = 'segm' if p.useSegm == 1 else 'bbox' - print('useSegm (deprecated) is not None. Running {} evaluation'.format(p.iouType)) + print('useSegm (deprecated) is not None. Running {} evaluation'. + format(p.iouType)) print('Evaluate annotation type *{}*'.format(p.iouType)) p.imgIds = list(np.unique(p.imgIds)) if p.useCats: p.catIds = list(np.unique(p.catIds)) p.maxDets = sorted(p.maxDets) - self.params=p + self.params = p self._prepare() # loop through images, area range, max detection number @@ -152,29 +158,28 @@ class COCOeval: evaluateImg = self.evaluateImg maxDet = p.maxDets[-1] - self.evalImgs = [evaluateImg(imgId, catId, areaRng, maxDet) - for catId in catIds - for areaRng in p.areaRng - for imgId in p.imgIds - ] + self.evalImgs = [ + evaluateImg(imgId, catId, areaRng, maxDet) for catId in catIds + for areaRng in p.areaRng for imgId in p.imgIds + ] self._paramsEval = copy.deepcopy(self.params) toc = time.time() - print('DONE (t={:0.2f}s).'.format(toc-tic)) + print('DONE (t={:0.2f}s).'.format(toc - tic)) def computeIoU(self, imgId, catId): p = self.params if p.useCats: - gt = self._gts[imgId,catId] - dt = self._dts[imgId,catId] + gt = self._gts[imgId, catId] + dt = self._dts[imgId, catId] else: - gt = [_ for cId in p.catIds for _ in self._gts[imgId,cId]] - dt = [_ for cId in p.catIds for _ in self._dts[imgId,cId]] - if len(gt) == 0 and len(dt) ==0: + gt = [_ for cId in p.catIds for _ in self._gts[imgId, cId]] + dt = [_ for cId in p.catIds for _ in self._dts[imgId, cId]] + if len(gt) == 0 and len(dt) == 0: return [] inds = np.argsort([-d['score'] for d in dt], kind='mergesort') dt = [dt[i] for i in inds] if len(dt) > p.maxDets[-1]: - dt=dt[0:p.maxDets[-1]] + dt = dt[0:p.maxDets[-1]] if p.iouType == 'segm': g = [g['segmentation'] for g in gt] @@ -187,7 +192,7 @@ class COCOeval: # compute iou between each dt and gt region iscrowd = [int(o['iscrowd']) for o in gt] - ious = iou(d,g,iscrowd) + ious = iou(d, g, iscrowd) return ious def computeOks(self, imgId, catId): @@ -203,33 +208,43 @@ class COCOeval: if len(gts) == 0 or len(dts) == 0: return [] ious = np.zeros((len(dts), len(gts))) - sigmas = np.array([.26, .25, .25, .35, .35, .79, .79, .72, .72, .62,.62, 1.07, 1.07, .87, .87, .89, .89])/10.0 + sigmas = np.array([ + .26, .25, .25, .35, .35, .79, .79, .72, .72, .62, .62, 1.07, 1.07, + .87, .87, .89, .89 + ]) / 10.0 vars = (sigmas * 2)**2 k = len(sigmas) # compute oks between each detection and ground truth object for j, gt in enumerate(gts): # create bounds for ignore regions(double the gt bbox) g = np.array(gt['keypoints']) - xg = g[0::3]; yg = g[1::3]; vg = g[2::3] + xg = g[0::3] + yg = g[1::3] + vg = g[2::3] k1 = np.count_nonzero(vg > 0) bb = gt['bbox'] - x0 = bb[0] - bb[2]; x1 = bb[0] + bb[2] * 2 - y0 = bb[1] - bb[3]; y1 = bb[1] + bb[3] * 2 + x0 = bb[0] - bb[2] + x1 = bb[0] + bb[2] * 2 + y0 = bb[1] - bb[3] + y1 = bb[1] + bb[3] * 2 for i, dt in enumerate(dts): d = np.array(dt['keypoints']) - xd = d[0::3]; yd = d[1::3] - if k1>0: + xd = d[0::3] + yd = d[1::3] + if k1 > 0: # measure the per-keypoint distance if keypoints visible dx = xd - xg dy = yd - yg else: # measure minimum distance to keypoints in (x0,y0) & (x1,y1) z = np.zeros((k)) - dx = np.max((z, x0-xd),axis=0)+np.max((z, xd-x1),axis=0) - dy = np.max((z, y0-yd),axis=0)+np.max((z, yd-y1),axis=0) - e = (dx**2 + dy**2) / vars / (gt['area']+np.spacing(1)) / 2 + dx = np.max((z, x0 - xd), axis=0) + np.max( + (z, xd - x1), axis=0) + dy = np.max((z, y0 - yd), axis=0) + np.max( + (z, yd - y1), axis=0) + e = (dx**2 + dy**2) / vars / (gt['area'] + np.spacing(1)) / 2 if k1 > 0: - e=e[vg > 0] + e = e[vg > 0] ious[i, j] = np.sum(np.exp(-e)) / e.shape[0] return ious @@ -240,16 +255,16 @@ class COCOeval: ''' p = self.params if p.useCats: - gt = self._gts[imgId,catId] - dt = self._dts[imgId,catId] + gt = self._gts[imgId, catId] + dt = self._dts[imgId, catId] else: - gt = [_ for cId in p.catIds for _ in self._gts[imgId,cId]] - dt = [_ for cId in p.catIds for _ in self._dts[imgId,cId]] - if len(gt) == 0 and len(dt) ==0: + gt = [_ for cId in p.catIds for _ in self._gts[imgId, cId]] + dt = [_ for cId in p.catIds for _ in self._dts[imgId, cId]] + if len(gt) == 0 and len(dt) == 0: return None for g in gt: - if g['ignore'] or (g['area']aRng[1]): + if g['ignore'] or (g['area'] < aRng[0] or g['area'] > aRng[1]): g['_ignore'] = 1 else: g['_ignore'] = 0 @@ -261,59 +276,62 @@ class COCOeval: dt = [dt[i] for i in dtind[0:maxDet]] iscrowd = [int(o['iscrowd']) for o in gt] # load computed ious - ious = self.ious[imgId, catId][:, gtind] if len(self.ious[imgId, catId]) > 0 else self.ious[imgId, catId] + ious = self.ious[imgId, catId][:, gtind] if len( + self.ious[imgId, catId]) > 0 else self.ious[imgId, catId] T = len(p.iouThrs) G = len(gt) D = len(dt) - gtm = np.zeros((T,G)) - dtm = np.zeros((T,D)) + gtm = np.zeros((T, G)) + dtm = np.zeros((T, D)) gtIg = np.array([g['_ignore'] for g in gt]) - dtIg = np.zeros((T,D)) - if not len(ious)==0: + dtIg = np.zeros((T, D)) + if not len(ious) == 0: for tind, t in enumerate(p.iouThrs): for dind, d in enumerate(dt): # information about best match so far (m=-1 -> unmatched) - iou = min([t,1-1e-10]) - m = -1 + iou = min([t, 1 - 1e-10]) + m = -1 for gind, g in enumerate(gt): # if this gt already matched, and not a crowd, continue - if gtm[tind,gind]>0 and not iscrowd[gind]: + if gtm[tind, gind] > 0 and not iscrowd[gind]: continue # if dt matched to reg gt, and on ignore gt, stop - if m>-1 and gtIg[m]==0 and gtIg[gind]==1: + if m > -1 and gtIg[m] == 0 and gtIg[gind] == 1: break # continue to next gt unless better match made - if ious[dind,gind] < iou: + if ious[dind, gind] < iou: continue # if match successful and best so far, store appropriately - iou=ious[dind,gind] - m=gind + iou = ious[dind, gind] + m = gind # if match made store id of match for both dt and gt - if m ==-1: + if m == -1: continue - dtIg[tind,dind] = gtIg[m] - dtm[tind,dind] = gt[m]['id'] - gtm[tind,m] = d['id'] + dtIg[tind, dind] = gtIg[m] + dtm[tind, dind] = gt[m]['id'] + gtm[tind, m] = d['id'] # set unmatched detections outside of area range to ignore - a = np.array([d['area']aRng[1] for d in dt]).reshape((1, len(dt))) - dtIg = np.logical_or(dtIg, np.logical_and(dtm==0, np.repeat(a,T,0))) + a = np.array([d['area'] < aRng[0] or d['area'] > aRng[1] + for d in dt]).reshape((1, len(dt))) + dtIg = np.logical_or(dtIg, np.logical_and(dtm == 0, np.repeat(a, T, + 0))) # store results for given image and category return { - 'image_id': imgId, - 'category_id': catId, - 'aRng': aRng, - 'maxDet': maxDet, - 'dtIds': [d['id'] for d in dt], - 'gtIds': [g['id'] for g in gt], - 'dtMatches': dtm, - 'gtMatches': gtm, - 'dtScores': [d['score'] for d in dt], - 'gtIgnore': gtIg, - 'dtIgnore': dtIg, - } + 'image_id': imgId, + 'category_id': catId, + 'aRng': aRng, + 'maxDet': maxDet, + 'dtIds': [d['id'] for d in dt], + 'gtIds': [g['id'] for g in gt], + 'dtMatches': dtm, + 'gtMatches': gtm, + 'dtScores': [d['score'] for d in dt], + 'gtIgnore': gtIg, + 'dtIgnore': dtIg, + } - def accumulate(self, p = None): + def accumulate(self, p=None): ''' Accumulate per image evaluation results and store the result in self.eval :param p: input params for evaluation @@ -327,13 +345,14 @@ class COCOeval: if p is None: p = self.params p.catIds = p.catIds if p.useCats == 1 else [-1] - T = len(p.iouThrs) - R = len(p.recThrs) - K = len(p.catIds) if p.useCats else 1 - A = len(p.areaRng) - M = len(p.maxDets) - precision = -np.ones((T,R,K,A,M)) # -1 for the precision of absent categories - recall = -np.ones((T,K,A,M)) + T = len(p.iouThrs) + R = len(p.recThrs) + K = len(p.catIds) if p.useCats else 1 + A = len(p.areaRng) + M = len(p.maxDets) + precision = -np.ones( + (T, R, K, A, M)) # -1 for the precision of absent categories + recall = -np.ones((T, K, A, M)) # create dictionary for future indexing _pe = self._paramsEval @@ -343,36 +362,45 @@ class COCOeval: setM = set(_pe.maxDets) setI = set(_pe.imgIds) # get inds to evaluate - k_list = [n for n, k in enumerate(p.catIds) if k in setK] + k_list = [n for n, k in enumerate(p.catIds) if k in setK] m_list = [m for n, m in enumerate(p.maxDets) if m in setM] - a_list = [n for n, a in enumerate(map(lambda x: tuple(x), p.areaRng)) if a in setA] - i_list = [n for n, i in enumerate(p.imgIds) if i in setI] + a_list = [ + n for n, a in enumerate(map(lambda x: tuple(x), p.areaRng)) + if a in setA + ] + i_list = [n for n, i in enumerate(p.imgIds) if i in setI] I0 = len(_pe.imgIds) A0 = len(_pe.areaRng) # retrieve E at each category, area range, and max number of detections for k, k0 in enumerate(k_list): - Nk = k0*A0*I0 + Nk = k0 * A0 * I0 for a, a0 in enumerate(a_list): - Na = a0*I0 + Na = a0 * I0 for m, maxDet in enumerate(m_list): E = [self.evalImgs[Nk + Na + i] for i in i_list] E = [e for e in E if not e is None] if len(E) == 0: continue - dtScores = np.concatenate([e['dtScores'][0:maxDet] for e in E]) + dtScores = np.concatenate( + [e['dtScores'][0:maxDet] for e in E]) # different sorting method generates slightly different results. # mergesort is used to be consistent as Matlab implementation. inds = np.argsort(-dtScores, kind='mergesort') - dtm = np.concatenate([e['dtMatches'][:,0:maxDet] for e in E], axis=1)[:,inds] - dtIg = np.concatenate([e['dtIgnore'][:,0:maxDet] for e in E], axis=1)[:,inds] + dtm = np.concatenate( + [e['dtMatches'][:, 0:maxDet] for e in E], axis=1)[:, + inds] + dtIg = np.concatenate( + [e['dtIgnore'][:, 0:maxDet] for e in E], axis=1)[:, + inds] gtIg = np.concatenate([e['gtIgnore'] for e in E]) - npig = np.count_nonzero(gtIg==0 ) + npig = np.count_nonzero(gtIg == 0) if npig == 0: continue - tps = np.logical_and( dtm, np.logical_not(dtIg) ) - fps = np.logical_and(np.logical_not(dtm), np.logical_not(dtIg) ) + tps = np.logical_and(dtm, np.logical_not(dtIg)) + fps = np.logical_and(np.logical_not(dtm), + np.logical_not(dtIg)) tp_sum = np.cumsum(tps, axis=1).astype(dtype=np.float) fp_sum = np.cumsum(fps, axis=1).astype(dtype=np.float) @@ -381,21 +409,22 @@ class COCOeval: fp = np.array(fp) nd = len(tp) rc = tp / npig - pr = tp / (fp+tp+np.spacing(1)) - q = np.zeros((R,)) + pr = tp / (fp + tp + np.spacing(1)) + q = np.zeros((R, )) if nd: - recall[t,k,a,m] = rc[-1] + recall[t, k, a, m] = rc[-1] else: - recall[t,k,a,m] = 0 + recall[t, k, a, m] = 0 # numpy is slow without cython optimization for accessing elements # use python array gets significant speed improvement - pr = pr.tolist(); q = q.tolist() + pr = pr.tolist() + q = q.tolist() - for i in range(nd-1, 0, -1): - if pr[i] > pr[i-1]: - pr[i-1] = pr[i] + for i in range(nd - 1, 0, -1): + if pr[i] > pr[i - 1]: + pr[i - 1] = pr[i] inds = np.searchsorted(rc, p.recThrs, side='left') try: @@ -403,31 +432,33 @@ class COCOeval: q[ri] = pr[pi] except: pass - precision[t,:,k,a,m] = np.array(q) + precision[t, :, k, a, m] = np.array(q) self.eval = { 'params': p, 'counts': [T, R, K, A, M], 'date': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), 'precision': precision, - 'recall': recall, + 'recall': recall, } toc = time.time() - print('DONE (t={:0.2f}s).'.format( toc-tic)) + print('DONE (t={:0.2f}s).'.format(toc - tic)) def summarize(self): ''' Compute and display summary metrics for evaluation results. Note this functin can *only* be applied on the default parameter setting ''' - def _summarize( ap=1, iouThr=None, areaRng='all', maxDets=100 ): + def _summarize(ap=1, iouThr=None, areaRng='all', maxDets=100): p = self.params iStr = ' {:<18} {} @[ IoU={:<9} | area={:>6s} | maxDets={:>3d} ] = {:0.3f}' titleStr = 'Average Precision' if ap == 1 else 'Average Recall' - typeStr = '(AP)' if ap==1 else '(AR)' + typeStr = '(AP)' if ap == 1 else '(AR)' iouStr = '{:0.2f}:{:0.2f}'.format(p.iouThrs[0], p.iouThrs[-1]) \ if iouThr is None else '{:0.2f}'.format(iouThr) - aind = [i for i, aRng in enumerate(p.areaRngLbl) if aRng == areaRng] + aind = [ + i for i, aRng in enumerate(p.areaRngLbl) if aRng == areaRng + ] mind = [i for i, mDet in enumerate(p.maxDets) if mDet == maxDets] if ap == 1: # dimension of precision: [TxRxKxAxM] @@ -436,37 +467,55 @@ class COCOeval: if iouThr is not None: t = np.where(iouThr == p.iouThrs)[0] s = s[t] - s = s[:,:,:,aind,mind] + s = s[:, :, :, aind, mind] else: # dimension of recall: [TxKxAxM] s = self.eval['recall'] if iouThr is not None: t = np.where(iouThr == p.iouThrs)[0] s = s[t] - s = s[:,:,aind,mind] - if len(s[s>-1])==0: + s = s[:, :, aind, mind] + if len(s[s > -1]) == 0: mean_s = -1 else: - mean_s = np.mean(s[s>-1]) - print(iStr.format(titleStr, typeStr, iouStr, areaRng, maxDets, mean_s)) + mean_s = np.mean(s[s > -1]) + print( + iStr.format(titleStr, typeStr, iouStr, areaRng, maxDets, + mean_s)) return mean_s + def _summarizeDets(): - stats = np.zeros((12,)) + stats = np.zeros((12, )) stats[0] = _summarize(1) stats[1] = _summarize(1, iouThr=.5, maxDets=self.params.maxDets[2]) - stats[2] = _summarize(1, iouThr=.75, maxDets=self.params.maxDets[2]) - stats[3] = _summarize(1, areaRng='small', maxDets=self.params.maxDets[2]) - stats[4] = _summarize(1, areaRng='medium', maxDets=self.params.maxDets[2]) - stats[5] = _summarize(1, areaRng='large', maxDets=self.params.maxDets[2]) + stats[2] = _summarize(1, + iouThr=.75, + maxDets=self.params.maxDets[2]) + stats[3] = _summarize(1, + areaRng='small', + maxDets=self.params.maxDets[2]) + stats[4] = _summarize(1, + areaRng='medium', + maxDets=self.params.maxDets[2]) + stats[5] = _summarize(1, + areaRng='large', + maxDets=self.params.maxDets[2]) stats[6] = _summarize(0, maxDets=self.params.maxDets[0]) stats[7] = _summarize(0, maxDets=self.params.maxDets[1]) stats[8] = _summarize(0, maxDets=self.params.maxDets[2]) - stats[9] = _summarize(0, areaRng='small', maxDets=self.params.maxDets[2]) - stats[10] = _summarize(0, areaRng='medium', maxDets=self.params.maxDets[2]) - stats[11] = _summarize(0, areaRng='large', maxDets=self.params.maxDets[2]) + stats[9] = _summarize(0, + areaRng='small', + maxDets=self.params.maxDets[2]) + stats[10] = _summarize(0, + areaRng='medium', + maxDets=self.params.maxDets[2]) + stats[11] = _summarize(0, + areaRng='large', + maxDets=self.params.maxDets[2]) return stats + def _summarizeKps(): - stats = np.zeros((10,)) + stats = np.zeros((10, )) stats[0] = _summarize(1, maxDets=20) stats[1] = _summarize(1, maxDets=20, iouThr=.5) stats[2] = _summarize(1, maxDets=20, iouThr=.75) @@ -478,6 +527,7 @@ class COCOeval: stats[8] = _summarize(0, maxDets=20, areaRng='medium') stats[9] = _summarize(0, maxDets=20, areaRng='large') return stats + if not self.eval: raise Exception('Please run accumulate() first') iouType = self.params.iouType @@ -490,6 +540,7 @@ class COCOeval: def __str__(self): self.summarize() + class Params: ''' Params for coco evaluation api @@ -498,10 +549,17 @@ class Params: self.imgIds = [] self.catIds = [] # np.arange causes trouble. the data point on arange is slightly larger than the true value - self.iouThrs = np.linspace(.5, 0.95, np.round((0.95 - .5) / .05) + 1, endpoint=True) - self.recThrs = np.linspace(.0, 1.00, np.round((1.00 - .0) / .01) + 1, endpoint=True) + self.iouThrs = np.linspace(.5, + 0.95, + np.round((0.95 - .5) / .05) + 1, + endpoint=True) + self.recThrs = np.linspace(.0, + 1.00, + np.round((1.00 - .0) / .01) + 1, + endpoint=True) self.maxDets = [1, 10, 100] - self.areaRng = [[0 ** 2, 1e5 ** 2], [0 ** 2, 32 ** 2], [32 ** 2, 96 ** 2], [96 ** 2, 1e5 ** 2]] + self.areaRng = [[0**2, 1e5**2], [0**2, 32**2], [32**2, 96**2], + [96**2, 1e5**2]] self.areaRngLbl = ['all', 'small', 'medium', 'large'] self.useCats = 1 @@ -509,10 +567,16 @@ class Params: self.imgIds = [] self.catIds = [] # np.arange causes trouble. the data point on arange is slightly larger than the true value - self.iouThrs = np.linspace(.5, 0.95, np.round((0.95 - .5) / .05) + 1, endpoint=True) - self.recThrs = np.linspace(.0, 1.00, np.round((1.00 - .0) / .01) + 1, endpoint=True) + self.iouThrs = np.linspace(.5, + 0.95, + np.round((0.95 - .5) / .05) + 1, + endpoint=True) + self.recThrs = np.linspace(.0, + 1.00, + np.round((1.00 - .0) / .01) + 1, + endpoint=True) self.maxDets = [20] - self.areaRng = [[0 ** 2, 1e5 ** 2], [32 ** 2, 96 ** 2], [96 ** 2, 1e5 ** 2]] + self.areaRng = [[0**2, 1e5**2], [32**2, 96**2], [96**2, 1e5**2]] self.areaRngLbl = ['all', 'medium', 'large'] self.useCats = 1 diff --git a/RetinaFace/rcnn/pycocotools/mask.py b/detection/RetinaFace/rcnn/pycocotools/mask.py similarity index 97% rename from RetinaFace/rcnn/pycocotools/mask.py rename to detection/RetinaFace/rcnn/pycocotools/mask.py index bba4ee0..3a9c14c 100644 --- a/RetinaFace/rcnn/pycocotools/mask.py +++ b/detection/RetinaFace/rcnn/pycocotools/mask.py @@ -73,10 +73,11 @@ from rcnn.pycocotools import _mask # Code written by Piotr Dollar and Tsung-Yi Lin, 2015. # Licensed under the Simplified BSD License [see coco/license.txt] -iou = _mask.iou -merge = _mask.merge +iou = _mask.iou +merge = _mask.merge frPyObjects = _mask.frPyObjects + def encode(bimask): if len(bimask.shape) == 3: return _mask.encode(bimask) @@ -84,11 +85,13 @@ def encode(bimask): h, w = bimask.shape return _mask.encode(bimask.reshape((h, w, 1), order='F'))[0] + def decode(rleObjs): if type(rleObjs) == list: return _mask.decode(rleObjs) else: - return _mask.decode([rleObjs])[:,:,0] + return _mask.decode([rleObjs])[:, :, 0] + def area(rleObjs): if type(rleObjs) == list: @@ -96,6 +99,7 @@ def area(rleObjs): else: return _mask.area([rleObjs])[0] + def toBbox(rleObjs): if type(rleObjs) == list: return _mask.toBbox(rleObjs) diff --git a/RetinaFace/rcnn/pycocotools/maskApi.c b/detection/RetinaFace/rcnn/pycocotools/maskApi.c similarity index 100% rename from RetinaFace/rcnn/pycocotools/maskApi.c rename to detection/RetinaFace/rcnn/pycocotools/maskApi.c diff --git a/RetinaFace/rcnn/pycocotools/maskApi.h b/detection/RetinaFace/rcnn/pycocotools/maskApi.h similarity index 100% rename from RetinaFace/rcnn/pycocotools/maskApi.h rename to detection/RetinaFace/rcnn/pycocotools/maskApi.h diff --git a/RetinaFace/rcnn/pycocotools/setup.py b/detection/RetinaFace/rcnn/pycocotools/setup.py similarity index 88% rename from RetinaFace/rcnn/pycocotools/setup.py rename to detection/RetinaFace/rcnn/pycocotools/setup.py index 5e836f1..d1e0d2e 100644 --- a/RetinaFace/rcnn/pycocotools/setup.py +++ b/detection/RetinaFace/rcnn/pycocotools/setup.py @@ -15,6 +15,4 @@ ext_modules = [ ) ] -setup(name='pycocotools', - ext_modules=cythonize(ext_modules) -) +setup(name='pycocotools', ext_modules=cythonize(ext_modules)) diff --git a/RetinaFace/rcnn/sample_config.py b/detection/RetinaFace/rcnn/sample_config.py similarity index 74% rename from RetinaFace/rcnn/sample_config.py rename to detection/RetinaFace/rcnn/sample_config.py index e07f77c..88fff9f 100644 --- a/RetinaFace/rcnn/sample_config.py +++ b/detection/RetinaFace/rcnn/sample_config.py @@ -11,50 +11,101 @@ config.IMAGE_STRIDE = 0 # dataset related params config.NUM_CLASSES = 2 -config.PRE_SCALES = [(1200, 1600)] # first is scale (the shorter side); second is max size -config.SCALES = [(640, 640)] # first is scale (the shorter side); second is max size +config.PRE_SCALES = [(1200, 1600) + ] # first is scale (the shorter side); second is max size +config.SCALES = [(640, 640) + ] # first is scale (the shorter side); second is max size #config.SCALES = [(800, 800)] # first is scale (the shorter side); second is max size config.ORIGIN_SCALE = False -_ratio = (1.,) +_ratio = (1., ) RAC_SSH = { - '32': {'SCALES': (32,16), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999}, - '16': {'SCALES': (8,4), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999}, - '8': {'SCALES': (2,1), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999}, + '32': { + 'SCALES': (32, 16), + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + }, + '16': { + 'SCALES': (8, 4), + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + }, + '8': { + 'SCALES': (2, 1), + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + }, } -_ratio = (1.,1.5) +_ratio = (1., 1.5) RAC_SSH2 = { - '32': {'SCALES': (32,16), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999}, - '16': {'SCALES': (8,4), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999}, - '8': {'SCALES': (2,1), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999}, + '32': { + 'SCALES': (32, 16), + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + }, + '16': { + 'SCALES': (8, 4), + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + }, + '8': { + 'SCALES': (2, 1), + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + }, } -_ratio = (1.,1.5) +_ratio = (1., 1.5) RAC_SSH3 = { - '32': {'SCALES': (32,16), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999}, - '16': {'SCALES': (8,4), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999}, - '8': {'SCALES': (2,1), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999}, - '4': {'SCALES': (2,1), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999}, + '32': { + 'SCALES': (32, 16), + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + }, + '16': { + 'SCALES': (8, 4), + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + }, + '8': { + 'SCALES': (2, 1), + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + }, + '4': { + 'SCALES': (2, 1), + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + }, } RAC_RETINA = {} -_ratios = (1.0,) -_ass = 2.0**(1.0/3) +_ratios = (1.0, ) +_ass = 2.0**(1.0 / 3) _basescale = 1.0 for _stride in [4, 8, 16, 32, 64]: - key = str(_stride) - value = {'BASE_SIZE': 16, 'RATIOS': _ratios, 'ALLOWED_BORDER': 9999} - scales = [] - for _ in range(3): - scales.append(_basescale) - _basescale *= _ass - value['SCALES'] = tuple(scales) - RAC_RETINA[key] = value + key = str(_stride) + value = {'BASE_SIZE': 16, 'RATIOS': _ratios, 'ALLOWED_BORDER': 9999} + scales = [] + for _ in range(3): + scales.append(_basescale) + _basescale *= _ass + value['SCALES'] = tuple(scales) + RAC_RETINA[key] = value - -config.RPN_ANCHOR_CFG = RAC_SSH #default +config.RPN_ANCHOR_CFG = RAC_SSH #default config.NET_MODE = 2 config.HEAD_MODULE = 'SSH' @@ -79,8 +130,8 @@ config.CASCADE = 0 config.CASCADE_MODE = 1 #config.CASCADE_CLS_STRIDES = [16,8,4] #config.CASCADE_BBOX_STRIDES = [64,32] -config.CASCADE_CLS_STRIDES = [64,32,16,8,4] -config.CASCADE_BBOX_STRIDES = [64,32,16,8,4] +config.CASCADE_CLS_STRIDES = [64, 32, 16, 8, 4] +config.CASCADE_BBOX_STRIDES = [64, 32, 16, 8, 4] #config.CASCADE_BBOX_STRIDES = [64,32,16,8] config.HEAD_BOX = False @@ -100,7 +151,6 @@ config.COLOR_JITTERING = 0.125 #config.COLOR_JITTERING = 0 #config.COLOR_JITTERING = 0.2 - config.TRAIN = edict() config.TRAIN.IMAGE_ALIGN = 0 @@ -123,7 +173,7 @@ config.TRAIN.RPN_BATCH_SIZE = 256 config.TRAIN.RPN_FG_FRACTION = 0.25 config.TRAIN.RPN_POSITIVE_OVERLAP = 0.5 config.TRAIN.RPN_NEGATIVE_OVERLAP = 0.3 -if config.CASCADE>0: +if config.CASCADE > 0: config.TRAIN.RPN_POSITIVE_OVERLAP = 0.7 config.TRAIN.CASCADE_OVERLAP = [0.4, 0.5] config.TRAIN.RPN_CLOBBER_POSITIVES = False @@ -132,7 +182,6 @@ config.TRAIN.RPN_FORCE_POSITIVE = False config.TRAIN.BBOX_STDS = (1.0, 1.0, 1.0, 1.0) config.TRAIN.LANDMARK_STD = 1.0 - config.TEST = edict() # R-CNN testing @@ -155,7 +204,6 @@ config.TEST.NMS = 0.3 config.TEST.SCORE_THRESH = 0.05 config.TEST.IOU_THRESH = 0.5 - # network settings network = edict() @@ -181,7 +229,7 @@ network.mnet.PIXEL_STDS = np.array([1.0, 1.0, 1.0]) network.mnet.PIXEL_SCALE = 1.0 #network.mnet.pretrained = 'model/mobilenetfd_0_25' #78 #network.mnet.pretrained = 'model/mobilenetfd2' #75 -network.mnet.pretrained = 'model/mobilenet025fd0' #78 +network.mnet.pretrained = 'model/mobilenet025fd0' #78 #network.mnet.pretrained = 'model/mobilenet025fd1' #75 #network.mnet.pretrained = 'model/mobilenet025fd2' # network.mnet.pretrained_epoch = 0 @@ -192,7 +240,6 @@ network.mnet.RPN_ANCHOR_CFG = RAC_SSH network.mnet.LAYER_FIX = True network.mnet.LANDMARK_LR_MULT = 2.5 - network.resnet = edict() #network.resnet.pretrained = 'model/ResNet50_v1d' #network.resnet.pretrained = 'model/resnet-50' @@ -222,7 +269,6 @@ network.resnet.USE_DCN = 0 network.resnet.pretrained = 'model/resnet-50' network.resnet.RPN_ANCHOR_CFG = RAC_SSH - # dataset settings dataset = edict() @@ -271,6 +317,7 @@ default.lr_step = '55,68,80' default.lr = 0.01 default.wd = 0.0005 + def generate_config(_network, _dataset): for k, v in network[_network].items(): if k in config: @@ -278,27 +325,27 @@ def generate_config(_network, _dataset): elif k in default: default[k] = v if k in config.TRAIN: - config.TRAIN[k] = v + config.TRAIN[k] = v for k, v in dataset[_dataset].items(): if k in config: config[k] = v elif k in default: default[k] = v if k in config.TRAIN: - config.TRAIN[k] = v + config.TRAIN[k] = v config.network = _network config.dataset = _dataset config.RPN_FEAT_STRIDE = [] num_anchors = [] for k in config.RPN_ANCHOR_CFG: - config.RPN_FEAT_STRIDE.append( int(k) ) - _num_anchors = len(config.RPN_ANCHOR_CFG[k]['SCALES'])*len(config.RPN_ANCHOR_CFG[k]['RATIOS']) - if config.DENSE_ANCHOR: - _num_anchors *= 2 - config.RPN_ANCHOR_CFG[k]['NUM_ANCHORS'] = _num_anchors - num_anchors.append(_num_anchors) + config.RPN_FEAT_STRIDE.append(int(k)) + _num_anchors = len(config.RPN_ANCHOR_CFG[k]['SCALES']) * len( + config.RPN_ANCHOR_CFG[k]['RATIOS']) + if config.DENSE_ANCHOR: + _num_anchors *= 2 + config.RPN_ANCHOR_CFG[k]['NUM_ANCHORS'] = _num_anchors + num_anchors.append(_num_anchors) config.RPN_FEAT_STRIDE = sorted(config.RPN_FEAT_STRIDE, reverse=True) - for j in range(1,len(num_anchors)): - assert num_anchors[0]==num_anchors[j] + for j in range(1, len(num_anchors)): + assert num_anchors[0] == num_anchors[j] config.NUM_ANCHORS = num_anchors[0] - diff --git a/RetinaFace/rcnn/symbol/__init__.py b/detection/RetinaFace/rcnn/symbol/__init__.py similarity index 100% rename from RetinaFace/rcnn/symbol/__init__.py rename to detection/RetinaFace/rcnn/symbol/__init__.py diff --git a/detection/RetinaFace/rcnn/symbol/pyramidbox.py b/detection/RetinaFace/rcnn/symbol/pyramidbox.py new file mode 100644 index 0000000..64ae7ce --- /dev/null +++ b/detection/RetinaFace/rcnn/symbol/pyramidbox.py @@ -0,0 +1,489 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np +import six +import paddle.fluid as fluid +from paddle.fluid.param_attr import ParamAttr +from paddle.fluid.initializer import Xavier +from paddle.fluid.initializer import Constant +from paddle.fluid.initializer import Bilinear +from paddle.fluid.regularizer import L2Decay + + +def conv_bn(input, + filter, + ksize, + stride, + padding, + act='relu', + bias_attr=False): + conv = fluid.layers.conv2d(input=input, + filter_size=ksize, + num_filters=filter, + stride=stride, + padding=padding, + act=None, + bias_attr=bias_attr) + return fluid.layers.batch_norm(input=conv, act=act) + + +def conv_block(input, groups, filters, ksizes, strides=None, with_pool=True): + assert len(filters) == groups + assert len(ksizes) == groups + strides = [1] * groups if strides is None else strides + w_attr = ParamAttr(learning_rate=1., initializer=Xavier()) + b_attr = ParamAttr(learning_rate=2., regularizer=L2Decay(0.)) + conv = input + for i in six.moves.xrange(groups): + conv = fluid.layers.conv2d(input=conv, + num_filters=filters[i], + filter_size=ksizes[i], + stride=strides[i], + padding=(ksizes[i] - 1) // 2, + param_attr=w_attr, + bias_attr=b_attr, + act='relu') + if with_pool: + pool = fluid.layers.pool2d(input=conv, + pool_size=2, + pool_type='max', + pool_stride=2, + ceil_mode=True) + return conv, pool + else: + return conv + + +class PyramidBox(object): + def __init__(self, + data_shape, + num_classes=None, + use_transposed_conv2d=True, + is_infer=False, + sub_network=False): + """ + TODO(qingqing): add comments. + """ + self.data_shape = data_shape + self.min_sizes = [16., 32., 64., 128., 256., 512.] + self.steps = [4., 8., 16., 32., 64., 128.] + self.num_classes = num_classes + self.use_transposed_conv2d = use_transposed_conv2d + self.is_infer = is_infer + self.sub_network = sub_network + + # the base network is VGG with atrous layers + self._input() + self._vgg() + if sub_network: + self._low_level_fpn() + self._cpm_module() + self._pyramidbox() + else: + self._vgg_ssd() + + def feeds(self): + if self.is_infer: + return [self.image] + else: + return [self.image, self.face_box, self.head_box, self.gt_label] + + def _input(self): + self.image = fluid.layers.data(name='image', + shape=self.data_shape, + dtype='float32') + if not self.is_infer: + self.face_box = fluid.layers.data(name='face_box', + shape=[4], + dtype='float32', + lod_level=1) + self.head_box = fluid.layers.data(name='head_box', + shape=[4], + dtype='float32', + lod_level=1) + self.gt_label = fluid.layers.data(name='gt_label', + shape=[1], + dtype='int32', + lod_level=1) + + def _vgg(self): + self.conv1, self.pool1 = conv_block(self.image, 2, [64] * 2, [3] * 2) + self.conv2, self.pool2 = conv_block(self.pool1, 2, [128] * 2, [3] * 2) + + #priorbox min_size is 16 + self.conv3, self.pool3 = conv_block(self.pool2, 3, [256] * 3, [3] * 3) + #priorbox min_size is 32 + self.conv4, self.pool4 = conv_block(self.pool3, 3, [512] * 3, [3] * 3) + #priorbox min_size is 64 + self.conv5, self.pool5 = conv_block(self.pool4, 3, [512] * 3, [3] * 3) + + # fc6 and fc7 in paper, priorbox min_size is 128 + self.conv6 = conv_block(self.pool5, + 2, [1024, 1024], [3, 1], + with_pool=False) + # conv6_1 and conv6_2 in paper, priorbox min_size is 256 + self.conv7 = conv_block(self.conv6, + 2, [256, 512], [1, 3], [1, 2], + with_pool=False) + # conv7_1 and conv7_2 in paper, priorbox mini_size is 512 + self.conv8 = conv_block(self.conv7, + 2, [128, 256], [1, 3], [1, 2], + with_pool=False) + + def _low_level_fpn(self): + """ + Low-level feature pyramid network. + """ + def fpn(up_from, up_to): + ch = up_to.shape[1] + b_attr = ParamAttr(learning_rate=2., regularizer=L2Decay(0.)) + conv1 = fluid.layers.conv2d(up_from, + ch, + 1, + act='relu', + bias_attr=b_attr) + if self.use_transposed_conv2d: + w_attr = ParamAttr(learning_rate=0., + regularizer=L2Decay(0.), + initializer=Bilinear()) + upsampling = fluid.layers.conv2d_transpose(conv1, + ch, + output_size=None, + filter_size=4, + padding=1, + stride=2, + groups=ch, + param_attr=w_attr, + bias_attr=False, + use_cudnn=True) + else: + upsampling = fluid.layers.resize_bilinear( + conv1, out_shape=up_to.shape[2:]) + + conv2 = fluid.layers.conv2d(up_to, + ch, + 1, + act='relu', + bias_attr=b_attr) + if self.is_infer: + upsampling = fluid.layers.crop(upsampling, shape=conv2) + # eltwise mul + conv_fuse = upsampling * conv2 + return conv_fuse + + self.lfpn2_on_conv5 = fpn(self.conv6, self.conv5) + self.lfpn1_on_conv4 = fpn(self.lfpn2_on_conv5, self.conv4) + self.lfpn0_on_conv3 = fpn(self.lfpn1_on_conv4, self.conv3) + + def _cpm_module(self): + """ + Context-sensitive Prediction Module + """ + def cpm(input): + # residual + branch1 = conv_bn(input, 1024, 1, 1, 0, None) + branch2a = conv_bn(input, 256, 1, 1, 0, act='relu') + branch2b = conv_bn(branch2a, 256, 3, 1, 1, act='relu') + branch2c = conv_bn(branch2b, 1024, 1, 1, 0, None) + sum = branch1 + branch2c + rescomb = fluid.layers.relu(x=sum) + + # ssh + b_attr = ParamAttr(learning_rate=2., regularizer=L2Decay(0.)) + ssh_1 = fluid.layers.conv2d(rescomb, + 256, + 3, + 1, + 1, + bias_attr=b_attr) + ssh_dimred = fluid.layers.conv2d(rescomb, + 128, + 3, + 1, + 1, + act='relu', + bias_attr=b_attr) + ssh_2 = fluid.layers.conv2d(ssh_dimred, + 128, + 3, + 1, + 1, + bias_attr=b_attr) + ssh_3a = fluid.layers.conv2d(ssh_dimred, + 128, + 3, + 1, + 1, + act='relu', + bias_attr=b_attr) + ssh_3b = fluid.layers.conv2d(ssh_3a, + 128, + 3, + 1, + 1, + bias_attr=b_attr) + + ssh_concat = fluid.layers.concat([ssh_1, ssh_2, ssh_3b], axis=1) + ssh_out = fluid.layers.relu(x=ssh_concat) + return ssh_out + + self.ssh_conv3 = cpm(self.lfpn0_on_conv3) + self.ssh_conv4 = cpm(self.lfpn1_on_conv4) + self.ssh_conv5 = cpm(self.lfpn2_on_conv5) + self.ssh_conv6 = cpm(self.conv6) + self.ssh_conv7 = cpm(self.conv7) + self.ssh_conv8 = cpm(self.conv8) + + def _l2_norm_scale(self, input, init_scale=1.0, channel_shared=False): + from paddle.fluid.layer_helper import LayerHelper + helper = LayerHelper("Scale") + l2_norm = fluid.layers.l2_normalize(input, + axis=1) # l2 norm along channel + shape = [1] if channel_shared else [input.shape[1]] + scale = helper.create_parameter( + attr=helper.param_attr, + shape=shape, + dtype=input.dtype, + default_initializer=Constant(init_scale)) + out = fluid.layers.elementwise_mul(x=l2_norm, + y=scale, + axis=-1 if channel_shared else 1) + return out + + def _pyramidbox(self): + """ + Get prior-boxes and pyramid-box + """ + self.ssh_conv3_norm = self._l2_norm_scale(self.ssh_conv3, + init_scale=10.) + self.ssh_conv4_norm = self._l2_norm_scale(self.ssh_conv4, + init_scale=8.) + self.ssh_conv5_norm = self._l2_norm_scale(self.ssh_conv5, + init_scale=5.) + + def permute_and_reshape(input, last_dim): + trans = fluid.layers.transpose(input, perm=[0, 2, 3, 1]) + compile_shape = [ + trans.shape[0], + np.prod(trans.shape[1:]) // last_dim, last_dim + ] + run_shape = fluid.layers.assign( + np.array([0, -1, last_dim]).astype("int32")) + return fluid.layers.reshape(trans, + shape=compile_shape, + actual_shape=run_shape) + + face_locs, face_confs = [], [] + head_locs, head_confs = [], [] + boxes, vars = [], [] + inputs = [ + self.ssh_conv3_norm, self.ssh_conv4_norm, self.ssh_conv5_norm, + self.ssh_conv6, self.ssh_conv7, self.ssh_conv8 + ] + b_attr = ParamAttr(learning_rate=2., regularizer=L2Decay(0.)) + for i, input in enumerate(inputs): + mbox_loc = fluid.layers.conv2d(input, 8, 3, 1, 1, bias_attr=b_attr) + face_loc, head_loc = fluid.layers.split(mbox_loc, + num_or_sections=2, + dim=1) + face_loc = permute_and_reshape(face_loc, 4) + head_loc = permute_and_reshape(head_loc, 4) + + mbox_conf = fluid.layers.conv2d(input, + 6, + 3, + 1, + 1, + bias_attr=b_attr) + face_conf1, face_conf3, head_conf = fluid.layers.split( + mbox_conf, num_or_sections=[1, 3, 2], dim=1) + face_conf3_maxin = fluid.layers.reduce_max(face_conf3, + dim=1, + keep_dim=True) + face_conf = fluid.layers.concat([face_conf1, face_conf3_maxin], + axis=1) + + face_conf = permute_and_reshape(face_conf, 2) + head_conf = permute_and_reshape(head_conf, 2) + + face_locs.append(face_loc) + face_confs.append(face_conf) + + head_locs.append(head_loc) + head_confs.append(head_conf) + + box, var = fluid.layers.prior_box(input, + self.image, + min_sizes=[self.min_sizes[i]], + steps=[self.steps[i]] * 2, + aspect_ratios=[1.], + clip=False, + flip=True, + offset=0.5) + box = fluid.layers.reshape(box, shape=[-1, 4]) + var = fluid.layers.reshape(var, shape=[-1, 4]) + + boxes.append(box) + vars.append(var) + + self.face_mbox_loc = fluid.layers.concat(face_locs, axis=1) + self.face_mbox_conf = fluid.layers.concat(face_confs, axis=1) + + self.head_mbox_loc = fluid.layers.concat(head_locs, axis=1) + self.head_mbox_conf = fluid.layers.concat(head_confs, axis=1) + + self.prior_boxes = fluid.layers.concat(boxes) + self.box_vars = fluid.layers.concat(vars) + + def _vgg_ssd(self): + self.conv3_norm = self._l2_norm_scale(self.conv3, init_scale=10.) + self.conv4_norm = self._l2_norm_scale(self.conv4, init_scale=8.) + self.conv5_norm = self._l2_norm_scale(self.conv5, init_scale=5.) + + def permute_and_reshape(input, last_dim): + trans = fluid.layers.transpose(input, perm=[0, 2, 3, 1]) + compile_shape = [ + trans.shape[0], + np.prod(trans.shape[1:]) // last_dim, last_dim + ] + run_shape = fluid.layers.assign( + np.array([0, -1, last_dim]).astype("int32")) + return fluid.layers.reshape(trans, + shape=compile_shape, + actual_shape=run_shape) + + locs, confs = [], [] + boxes, vars = [], [] + b_attr = ParamAttr(learning_rate=2., regularizer=L2Decay(0.)) + + # conv3 + mbox_loc = fluid.layers.conv2d(self.conv3_norm, + 4, + 3, + 1, + 1, + bias_attr=b_attr) + loc = permute_and_reshape(mbox_loc, 4) + mbox_conf = fluid.layers.conv2d(self.conv3_norm, + 4, + 3, + 1, + 1, + bias_attr=b_attr) + conf1, conf3 = fluid.layers.split(mbox_conf, + num_or_sections=[1, 3], + dim=1) + conf3_maxin = fluid.layers.reduce_max(conf3, dim=1, keep_dim=True) + conf = fluid.layers.concat([conf1, conf3_maxin], axis=1) + conf = permute_and_reshape(conf, 2) + box, var = fluid.layers.prior_box(self.conv3_norm, + self.image, + min_sizes=[16.], + steps=[4, 4], + aspect_ratios=[1.], + clip=False, + flip=True, + offset=0.5) + box = fluid.layers.reshape(box, shape=[-1, 4]) + var = fluid.layers.reshape(var, shape=[-1, 4]) + + locs.append(loc) + confs.append(conf) + boxes.append(box) + vars.append(var) + + min_sizes = [32., 64., 128., 256., 512.] + steps = [8., 16., 32., 64., 128.] + inputs = [ + self.conv4_norm, self.conv5_norm, self.conv6, self.conv7, + self.conv8 + ] + for i, input in enumerate(inputs): + mbox_loc = fluid.layers.conv2d(input, 4, 3, 1, 1, bias_attr=b_attr) + loc = permute_and_reshape(mbox_loc, 4) + + mbox_conf = fluid.layers.conv2d(input, + 2, + 3, + 1, + 1, + bias_attr=b_attr) + conf = permute_and_reshape(mbox_conf, 2) + box, var = fluid.layers.prior_box(input, + self.image, + min_sizes=[min_sizes[i]], + steps=[steps[i]] * 2, + aspect_ratios=[1.], + clip=False, + flip=True, + offset=0.5) + box = fluid.layers.reshape(box, shape=[-1, 4]) + var = fluid.layers.reshape(var, shape=[-1, 4]) + + locs.append(loc) + confs.append(conf) + boxes.append(box) + vars.append(var) + + self.face_mbox_loc = fluid.layers.concat(locs, axis=1) + self.face_mbox_conf = fluid.layers.concat(confs, axis=1) + self.prior_boxes = fluid.layers.concat(boxes) + self.box_vars = fluid.layers.concat(vars) + + def vgg_ssd_loss(self): + loss = fluid.layers.ssd_loss(self.face_mbox_loc, + self.face_mbox_conf, + self.face_box, + self.gt_label, + self.prior_boxes, + self.box_vars, + overlap_threshold=0.35, + neg_overlap=0.35) + loss = fluid.layers.reduce_sum(loss) + return loss + + def train(self): + face_loss = fluid.layers.ssd_loss(self.face_mbox_loc, + self.face_mbox_conf, + self.face_box, + self.gt_label, + self.prior_boxes, + self.box_vars, + overlap_threshold=0.35, + neg_overlap=0.35) + face_loss.persistable = True + head_loss = fluid.layers.ssd_loss(self.head_mbox_loc, + self.head_mbox_conf, + self.head_box, + self.gt_label, + self.prior_boxes, + self.box_vars, + overlap_threshold=0.35, + neg_overlap=0.35) + head_loss.persistable = True + face_loss = fluid.layers.reduce_sum(face_loss) + face_loss.persistable = True + head_loss = fluid.layers.reduce_sum(head_loss) + head_loss.persistable = True + total_loss = face_loss + head_loss + total_loss.persistable = True + return face_loss, head_loss, total_loss + + def infer(self, main_program=None): + if main_program is None: + test_program = fluid.default_main_program().clone(for_test=True) + else: + test_program = main_program.clone(for_test=True) + with fluid.program_guard(test_program): + face_nmsed_out = fluid.layers.detection_output( + self.face_mbox_loc, + self.face_mbox_conf, + self.prior_boxes, + self.box_vars, + nms_threshold=0.3, + nms_top_k=5000, + keep_top_k=750, + score_threshold=0.01) + return test_program, face_nmsed_out diff --git a/detection/RetinaFace/rcnn/symbol/symbol_common.py b/detection/RetinaFace/rcnn/symbol/symbol_common.py new file mode 100644 index 0000000..1343dd9 --- /dev/null +++ b/detection/RetinaFace/rcnn/symbol/symbol_common.py @@ -0,0 +1,757 @@ +import mxnet as mx +import mxnet.ndarray as nd +import numpy as np +from rcnn.config import config +from rcnn.PY_OP import rpn_fpn_ohem3, cascade_refine + +PREFIX = 'RF' +F1 = 0 +F2 = 0 +_bwm = 1.0 + +def conv_only(from_layer, name, num_filter, kernel=(1,1), pad=(0,0), \ + stride=(1,1), bias_wd_mult=0.0, shared_weight=None, shared_bias = None): + if shared_weight is None: + weight = mx.symbol.Variable(name="{}_weight".format(name), + init=mx.init.Normal(0.01), + attr={'__lr_mult__': '1.0'}) + bias = mx.symbol.Variable(name="{}_bias".format(name), + init=mx.init.Constant(0.0), + attr={ + '__lr_mult__': '2.0', + '__wd_mult__': str(bias_wd_mult) + }) + else: + weight = shared_weight + bias = shared_bias + print('reuse shared var in', name) + conv = mx.symbol.Convolution(data=from_layer, kernel=kernel, pad=pad, \ + stride=stride, num_filter=num_filter, name="{}".format(name), weight = weight, bias=bias) + return conv + + +def conv_deformable(net, num_filter, num_group=1, act_type='relu', name=''): + if config.USE_DCN == 1: + f = num_group * 18 + conv_offset = mx.symbol.Convolution(name=name + '_conv_offset', + data=net, + num_filter=f, + pad=(1, 1), + kernel=(3, 3), + stride=(1, 1)) + net = mx.contrib.symbol.DeformableConvolution( + name=name + "_conv", + data=net, + offset=conv_offset, + num_filter=num_filter, + pad=(1, 1), + kernel=(3, 3), + num_deformable_group=num_group, + stride=(1, 1), + no_bias=False) + else: + print('use dcnv2 at', name) + lr_mult = 0.1 + weight_var = mx.sym.Variable(name=name + '_conv2_offset_weight', + init=mx.init.Zero(), + lr_mult=lr_mult) + bias_var = mx.sym.Variable(name=name + '_conv2_offset_bias', + init=mx.init.Zero(), + lr_mult=lr_mult) + conv2_offset = mx.symbol.Convolution(name=name + '_conv2_offset', + data=net, + num_filter=27, + pad=(1, 1), + kernel=(3, 3), + stride=(1, 1), + weight=weight_var, + bias=bias_var, + lr_mult=lr_mult) + conv2_offset_t = mx.sym.slice_axis(conv2_offset, + axis=1, + begin=0, + end=18) + conv2_mask = mx.sym.slice_axis(conv2_offset, + axis=1, + begin=18, + end=None) + conv2_mask = 2 * mx.sym.Activation(conv2_mask, act_type='sigmoid') + + conv2 = mx.contrib.symbol.ModulatedDeformableConvolution( + name=name + '_conv2', + data=net, + offset=conv2_offset_t, + mask=conv2_mask, + num_filter=num_filter, + pad=(1, 1), + kernel=(3, 3), + stride=(1, 1), + num_deformable_group=num_group, + no_bias=True) + net = conv2 + net = mx.sym.BatchNorm(data=net, + fix_gamma=False, + eps=2e-5, + momentum=0.9, + name=name + '_bn') + if len(act_type) > 0: + net = mx.symbol.Activation(data=net, + act_type=act_type, + name=name + '_act') + return net + +def conv_act_layer_dw(from_layer, name, num_filter, kernel=(1,1), pad=(0,0), \ + stride=(1,1), act_type="relu", bias_wd_mult=0.0): + assert kernel[0] == 3 + weight = mx.symbol.Variable(name="{}_weight".format(name), + init=mx.init.Normal(0.01), + attr={'__lr_mult__': '1.0'}) + bias = mx.symbol.Variable(name="{}_bias".format(name), + init=mx.init.Constant(0.0), + attr={ + '__lr_mult__': '2.0', + '__wd_mult__': str(bias_wd_mult) + }) + conv = mx.symbol.Convolution(data=from_layer, kernel=kernel, pad=pad, \ + stride=stride, num_filter=num_filter, num_group=num_filter, name="{}".format(name), weight=weight, bias=bias) + conv = mx.sym.BatchNorm(data=conv, + fix_gamma=False, + eps=2e-5, + momentum=0.9, + name=name + '_bn') + if len(act_type) > 0: + relu = mx.symbol.Activation(data=conv, act_type=act_type, \ + name="{}_{}".format(name, act_type)) + else: + relu = conv + return relu + +def conv_act_layer(from_layer, name, num_filter, kernel=(1,1), pad=(0,0), \ + stride=(1,1), act_type="relu", bias_wd_mult=0.0, separable=False, filter_in = -1): + + if config.USE_DCN > 1 and kernel == (3, 3) and pad == ( + 1, 1) and stride == (1, 1) and not separable: + return conv_deformable(from_layer, + num_filter, + num_group=1, + act_type=act_type, + name=name) + + if separable: + assert kernel[0] > 1 + assert filter_in > 0 + if not separable: + weight = mx.symbol.Variable(name="{}_weight".format(name), + init=mx.init.Normal(0.01), + attr={'__lr_mult__': '1.0'}) + bias = mx.symbol.Variable(name="{}_bias".format(name), + init=mx.init.Constant(0.0), + attr={ + '__lr_mult__': '2.0', + '__wd_mult__': str(bias_wd_mult) + }) + conv = mx.symbol.Convolution(data=from_layer, kernel=kernel, pad=pad, \ + stride=stride, num_filter=num_filter, name="{}".format(name), weight=weight, bias=bias) + conv = mx.sym.BatchNorm(data=conv, + fix_gamma=False, + eps=2e-5, + momentum=0.9, + name=name + '_bn') + else: + if filter_in < 0: + filter_in = num_filter + conv = mx.symbol.Convolution(data=from_layer, kernel=kernel, pad=pad, \ + stride=stride, num_filter=filter_in, num_group=filter_in, name="{}_sep".format(name)) + conv = mx.sym.BatchNorm(data=conv, + fix_gamma=False, + eps=2e-5, + momentum=0.9, + name=name + '_sep_bn') + conv = mx.symbol.Activation(data=conv, act_type='relu', \ + name="{}_sep_bn_relu".format(name)) + conv = mx.symbol.Convolution(data=conv, kernel=(1,1), pad=(0,0), \ + stride=(1,1), num_filter=num_filter, name="{}".format(name)) + conv = mx.sym.BatchNorm(data=conv, + fix_gamma=False, + eps=2e-5, + momentum=0.9, + name=name + '_bn') + if len(act_type) > 0: + relu = mx.symbol.Activation(data=conv, act_type=act_type, \ + name="{}_{}".format(name, act_type)) + else: + relu = conv + return relu + + +def ssh_context_module(body, num_filter, filter_in, name): + conv_dimred = conv_act_layer(body, + name + '_conv1', + num_filter, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + separable=False, + filter_in=filter_in) + conv5x5 = conv_act_layer(conv_dimred, + name + '_conv2', + num_filter, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='', + separable=False) + conv7x7_1 = conv_act_layer(conv_dimred, + name + '_conv3_1', + num_filter, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + separable=False) + conv7x7 = conv_act_layer(conv7x7_1, + name + '_conv3_2', + num_filter, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='', + separable=False) + return (conv5x5, conv7x7) + + +def ssh_detection_module(body, num_filter, filter_in, name): + assert num_filter % 4 == 0 + conv3x3 = conv_act_layer(body, + name + '_conv1', + num_filter // 2, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='', + separable=False, + filter_in=filter_in) + #_filter = max(num_filter//4, 16) + _filter = num_filter // 4 + conv5x5, conv7x7 = ssh_context_module(body, _filter, filter_in, + name + '_context') + ret = mx.sym.concat(*[conv3x3, conv5x5, conv7x7], + dim=1, + name=name + '_concat') + ret = mx.symbol.Activation(data=ret, + act_type='relu', + name=name + '_concat_relu') + out_filter = num_filter // 2 + _filter * 2 + if config.USE_DCN > 0: + ret = conv_deformable(ret, + num_filter=out_filter, + name=name + '_concat_dcn') + return ret + + +#def retina_context_module(body, kernel, num_filter, filter_in, name): +# conv_dimred = conv_act_layer(body, name+'_conv0', +# num_filter, kernel=(1,1), pad=(0,0), stride=(1, 1), act_type='relu', separable=False, filter_in = filter_in) +# conv1 = conv_act_layer(conv_dimred, name+'_conv1', +# num_filter*6, kernel=(1,1), pad=(0,0), stride=(1, 1), act_type='relu', separable=False, filter_in = filter_in) +# conv2 = conv_act_layer(conv1, name+'_conv2', +# num_filter*6, kernel=kernel, pad=((kernel[0]-1)//2, (kernel[1]-1)//2), stride=(1, 1), act_type='relu', separable=True, filter_in = num_filter*6) +# conv3 = conv_act_layer(conv2, name+'_conv3', +# num_filter, kernel=(1,1), pad=(0,0), stride=(1, 1), act_type='relu', separable=False) +# conv3 = conv3 + conv_dimred +# return conv3 + + +def retina_detection_module(body, num_filter, filter_in, name): + assert num_filter % 4 == 0 + conv1 = conv_act_layer(body, + name + '_conv1', + num_filter // 2, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + separable=False, + filter_in=filter_in) + conv2 = conv_act_layer(conv1, + name + '_conv2', + num_filter // 2, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + separable=False, + filter_in=num_filter // 2) + conv3 = conv_act_layer(conv2, + name + '_conv3', + num_filter // 2, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + separable=False, + filter_in=num_filter // 2) + conv4 = conv2 + conv3 + body = mx.sym.concat(*[conv1, conv4], dim=1, name=name + '_concat') + if config.USE_DCN > 0: + body = conv_deformable(body, + num_filter=num_filter, + name=name + '_concat_dcn') + return body + + +def head_module(body, num_filter, filter_in, name): + if config.HEAD_MODULE == 'SSH': + return ssh_detection_module(body, num_filter, filter_in, name) + else: + return retina_detection_module(body, num_filter, filter_in, name) + + +def upsampling(data, num_filter, name): + #ret = mx.symbol.Deconvolution(data=data, num_filter=num_filter, kernel=(4,4), stride=(2, 2), pad=(1,1), + # num_group = num_filter, no_bias = True, attr={'__lr_mult__': '0.0', '__wd_mult__': '0.0'}, + # name=name) + #ret = mx.symbol.Deconvolution(data=data, num_filter=num_filter, kernel=(2,2), stride=(2, 2), pad=(0,0), + # num_group = num_filter, no_bias = True, attr={'__lr_mult__': '0.0', '__wd_mult__': '0.0'}, + # name=name) + ret = mx.symbol.UpSampling(data, + scale=2, + sample_type='nearest', + workspace=512, + name=name, + num_args=1) + return ret + + +def get_sym_by_name(name, sym_buffer): + if name in sym_buffer: + return sym_buffer[name] + ret = None + name_key = name[0:1] + name_num = int(name[1:]) + #print('getting', name, name_key, name_num) + if name_key == 'C': + assert name_num % 2 == 0 + bottom = get_sym_by_name('C%d' % (name_num // 2), sym_buffer) + ret = conv_act_layer(bottom, + '%s_C%d' (PREFIX, name_num), + F1, + kernel=(3, 3), + pad=(1, 1), + stride=(2, 2), + act_type='relu', + bias_wd_mult=_bwm) + elif name_key == 'P': + assert name_num % 2 == 0 + assert name_num <= max(config.RPN_FEAT_STRIDE) + lateral = get_sym_by_name('L%d' % (name_num), sym_buffer) + if name_num == max(config.RPN_FEAT_STRIDE) or name_num > 32: + ret = mx.sym.identity(lateral, name='%s_P%d' % (PREFIX, name_num)) + else: + bottom = get_sym_by_name('L%d' % (name_num * 2), sym_buffer) + bottom_up = upsampling(bottom, F1, '%s_U%d' % (PREFIX, name_num)) + if config.USE_CROP: + bottom_up = mx.symbol.Crop(*[bottom_up, lateral]) + aggr = lateral + bottom_up + aggr = conv_act_layer(aggr, + '%s_A%d' % (PREFIX, name_num), + F1, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + ret = mx.sym.identity(aggr, name='%s_P%d' % (PREFIX, name_num)) + elif name_key == 'L': + c = get_sym_by_name('C%d' % (name_num), sym_buffer) + #print('L', name, F1) + ret = conv_act_layer(c, + '%s_L%d' % (PREFIX, name_num), + F1, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + else: + raise RuntimeError('%s is not a valid sym key name' % name) + sym_buffer[name] = ret + return ret + + +def get_sym_conv(data, sym): + all_layers = sym.get_internals() + + isize = 640 + _, out_shape, _ = all_layers.infer_shape(data=(1, 3, isize, isize)) + last_entry = None + c1 = None + c2 = None + c3 = None + c1_name = None + c2_name = None + c3_name = None + c1_filter = -1 + c2_filter = -1 + c3_filter = -1 + #print(len(all_layers), len(out_shape)) + #print(all_layers.__class__) + outputs = all_layers.list_outputs() + #print(outputs.__class__, len(outputs)) + count = len(outputs) + stride2name = {} + stride2layer = {} + stride2shape = {} + for i in range(count): + name = outputs[i] + shape = out_shape[i] + print(i, name, count, shape) + if not name.endswith('_output'): + continue + if len(shape) != 4: + continue + assert isize % shape[2] == 0 + if shape[1] > config.max_feat_channel: + break + stride = isize // shape[2] + stride2name[stride] = name + stride2layer[stride] = all_layers[name] + stride2shape[stride] = shape + + strides = sorted(stride2name.keys()) + for stride in strides: + print('stride', stride, stride2name[stride], stride2shape[stride]) + print('F1_F2', F1, F2) + #print('cnames', c1_name, c2_name, c3_name, F1, F2) + _bwm = 1.0 + ret = {} + sym_buffer = {} + for stride in [4, 8, 16, 32]: + sym_buffer['C%d' % stride] = stride2layer[stride] + if not config.USE_FPN: + for stride in config.RPN_FEAT_STRIDE: + name = 'L%d' % stride + ret[stride] = get_sym_by_name(name, sym_buffer) + else: + for stride in config.RPN_FEAT_STRIDE: + name = 'P%d' % stride + ret[stride] = get_sym_by_name(name, sym_buffer) + + return ret + + +def get_out(conv_fpn_feat, + prefix, + stride, + landmark=False, + lr_mult=1.0, + gt_boxes=None): + A = config.NUM_ANCHORS + bbox_pred_len = 4 + landmark_pred_len = 10 + if config.USE_BLUR: + bbox_pred_len = 5 + if config.USE_OCCLUSION: + landmark_pred_len = 15 + ret_group = [] + num_anchors = config.RPN_ANCHOR_CFG[str(stride)]['NUM_ANCHORS'] + cls_label = mx.symbol.Variable(name='%s_label_stride%d' % (prefix, stride)) + bbox_target = mx.symbol.Variable(name='%s_bbox_target_stride%d' % + (prefix, stride)) + bbox_weight = mx.symbol.Variable(name='%s_bbox_weight_stride%d' % + (prefix, stride)) + if landmark: + landmark_target = mx.symbol.Variable( + name='%s_landmark_target_stride%d' % (prefix, stride)) + landmark_weight = mx.symbol.Variable( + name='%s_landmark_weight_stride%d' % (prefix, stride)) + conv_feat = conv_fpn_feat[stride] + rpn_relu = head_module(conv_feat, F2 * config.CONTEXT_FILTER_RATIO, F1, + 'rf_head_stride%d' % stride) + + rpn_cls_score = conv_only(rpn_relu, + '%s_rpn_cls_score_stride%d' % (prefix, stride), + 2 * num_anchors, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1)) + + rpn_bbox_pred = conv_only(rpn_relu, + '%s_rpn_bbox_pred_stride%d' % (prefix, stride), + bbox_pred_len * num_anchors, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1)) + + # prepare rpn data + rpn_cls_score_reshape = mx.symbol.Reshape( + data=rpn_cls_score, + shape=(0, 2, -1), + name="%s_rpn_cls_score_reshape_stride%s" % (prefix, stride)) + + rpn_bbox_pred_reshape = mx.symbol.Reshape( + data=rpn_bbox_pred, + shape=(0, 0, -1), + name="%s_rpn_bbox_pred_reshape_stride%s" % (prefix, stride)) + if landmark: + rpn_landmark_pred = conv_only(rpn_relu, + '%s_rpn_landmark_pred_stride%d' % + (prefix, stride), + landmark_pred_len * num_anchors, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1)) + rpn_landmark_pred_reshape = mx.symbol.Reshape( + data=rpn_landmark_pred, + shape=(0, 0, -1), + name="%s_rpn_landmark_pred_reshape_stride%s" % (prefix, stride)) + + if config.TRAIN.RPN_ENABLE_OHEM >= 2: + label, anchor_weight, pos_count = mx.sym.Custom( + op_type='rpn_fpn_ohem3', + stride=int(stride), + network=config.network, + dataset=config.dataset, + prefix=prefix, + cls_score=rpn_cls_score_reshape, + labels=cls_label) + + _bbox_weight = mx.sym.tile(anchor_weight, (1, 1, bbox_pred_len)) + _bbox_weight = _bbox_weight.reshape( + (0, -1, A * bbox_pred_len)).transpose((0, 2, 1)) + bbox_weight = mx.sym.elemwise_mul(bbox_weight, + _bbox_weight, + name='%s_bbox_weight_mul_stride%s' % + (prefix, stride)) + + if landmark: + _landmark_weight = mx.sym.tile(anchor_weight, + (1, 1, landmark_pred_len)) + _landmark_weight = _landmark_weight.reshape( + (0, -1, A * landmark_pred_len)).transpose((0, 2, 1)) + landmark_weight = mx.sym.elemwise_mul( + landmark_weight, + _landmark_weight, + name='%s_landmark_weight_mul_stride%s' % (prefix, stride)) + else: + label = cls_label + #if not config.FACE_LANDMARK: + # label, bbox_weight = mx.sym.Custom(op_type='rpn_fpn_ohem', stride=int(stride), cls_score=rpn_cls_score_reshape, bbox_weight = bbox_weight , labels = label) + #else: + # label, bbox_weight, landmark_weight = mx.sym.Custom(op_type='rpn_fpn_ohem2', stride=int(stride), cls_score=rpn_cls_score_reshape, bbox_weight = bbox_weight, landmark_weight=landmark_weight, labels = label) + #cls loss + rpn_cls_prob = mx.symbol.SoftmaxOutput(data=rpn_cls_score_reshape, + label=label, + multi_output=True, + normalization='valid', + use_ignore=True, + ignore_label=-1, + grad_scale=lr_mult, + name='%s_rpn_cls_prob_stride%d' % + (prefix, stride)) + ret_group.append(rpn_cls_prob) + ret_group.append(mx.sym.BlockGrad(label)) + + pos_count = mx.symbol.sum(pos_count) + pos_count = pos_count + 0.001 #avoid zero + + #bbox loss + bbox_diff = rpn_bbox_pred_reshape - bbox_target + bbox_diff = bbox_diff * bbox_weight + rpn_bbox_loss_ = mx.symbol.smooth_l1(name='%s_rpn_bbox_loss_stride%d_' % + (prefix, stride), + scalar=3.0, + data=bbox_diff) + bbox_lr_mode0 = 0.25 * lr_mult * config.TRAIN.BATCH_IMAGES / config.TRAIN.RPN_BATCH_SIZE + landmark_lr_mode0 = 0.4 * config.LANDMARK_LR_MULT * bbox_lr_mode0 + if config.LR_MODE == 0: + rpn_bbox_loss = mx.sym.MakeLoss(name='%s_rpn_bbox_loss_stride%d' % + (prefix, stride), + data=rpn_bbox_loss_, + grad_scale=bbox_lr_mode0) + else: + rpn_bbox_loss_ = mx.symbol.broadcast_div(rpn_bbox_loss_, pos_count) + rpn_bbox_loss = mx.sym.MakeLoss(name='%s_rpn_bbox_loss_stride%d' % + (prefix, stride), + data=rpn_bbox_loss_, + grad_scale=0.5 * lr_mult) + ret_group.append(rpn_bbox_loss) + ret_group.append(mx.sym.BlockGrad(bbox_weight)) + + #landmark loss + if landmark: + landmark_diff = rpn_landmark_pred_reshape - landmark_target + landmark_diff = landmark_diff * landmark_weight + rpn_landmark_loss_ = mx.symbol.smooth_l1( + name='%s_rpn_landmark_loss_stride%d_' % (prefix, stride), + scalar=3.0, + data=landmark_diff) + if config.LR_MODE == 0: + rpn_landmark_loss = mx.sym.MakeLoss( + name='%s_rpn_landmark_loss_stride%d' % (prefix, stride), + data=rpn_landmark_loss_, + grad_scale=landmark_lr_mode0) + else: + rpn_landmark_loss_ = mx.symbol.broadcast_div( + rpn_landmark_loss_, pos_count) + rpn_landmark_loss = mx.sym.MakeLoss( + name='%s_rpn_landmark_loss_stride%d' % (prefix, stride), + data=rpn_landmark_loss_, + grad_scale=0.2 * config.LANDMARK_LR_MULT * lr_mult) + ret_group.append(rpn_landmark_loss) + ret_group.append(mx.sym.BlockGrad(landmark_weight)) + if config.USE_3D: + from rcnn.PY_OP import rpn_3d_mesh + pass + if config.CASCADE > 0: + if config.CASCADE_MODE == 0: + body = rpn_relu + elif config.CASCADE_MODE == 1: + body = head_module(conv_feat, F2 * config.CONTEXT_FILTER_RATIO, F1, + '%s_head_stride%d_cas' % (PREFIX, stride)) + elif config.CASCADE_MODE == 2: + body = conv_feat + rpn_relu + body = head_module(body, F2 * config.CONTEXT_FILTER_RATIO, F1, + '%s_head_stride%d_cas' % (PREFIX, stride)) + else: + body = head_module(conv_feat, F2 * config.CONTEXT_FILTER_RATIO, F1, + '%s_head_stride%d_cas' % (PREFIX, stride)) + body = mx.sym.concat(body, + rpn_cls_score, + rpn_bbox_pred, + rpn_landmark_pred, + dim=1) + + #cls_pred = rpn_cls_prob + cls_pred_t0 = rpn_cls_score_reshape + cls_label_raw = cls_label + cls_label_t0 = label + bbox_pred_t0 = rpn_bbox_pred_reshape + #bbox_pred = rpn_bbox_pred + #bbox_pred = mx.sym.transpose(bbox_pred, (0, 2, 3, 1)) + #bbox_pred_len = 4 + #bbox_pred = mx.sym.reshape(bbox_pred, (0, -1, bbox_pred_len)) + bbox_label_t0 = bbox_target + #prefix = prefix+'2' + for casid in range(config.CASCADE): + #pseudo-code + #anchor_label = GENANCHOR(bbox_label, bbox_pred, stride) + #bbox_label = F(anchor_label, bbox_pred) + #bbox_label = bbox_label - bbox_pred + cls_pred = conv_only(body, + '%s_rpn_cls_score_stride%d_cas%d' % + (prefix, stride, casid), + 2 * num_anchors, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1)) + rpn_cls_score_reshape = mx.symbol.Reshape( + data=cls_pred, + shape=(0, 2, -1), + name="%s_rpn_cls_score_reshape_stride%s_cas%d" % + (prefix, stride, casid)) + + #bbox_label equals to bbox_target + #cls_pred, cls_label, bbox_pred, bbox_label, bbox_weight, pos_count = mx.sym.Custom(op_type='cascade_refine', stride=int(stride), network=config.network, dataset=config.dataset, prefix=prefix, cls_pred=cls_pred, cls_label = cls_label, bbox_pred = bbox_pred, bbox_label = bbox_label) + #cls_label, bbox_label, anchor_weight, pos_count = mx.sym.Custom(op_type='cascade_refine', stride=int(stride), network=config.network, dataset=config.dataset, prefix=prefix, cls_pred_t0=cls_pred_t0, cls_label_t0 = cls_label_t0, cls_pred = rpn_cls_score_reshape, bbox_pred_t0 = bbox_pred_t0, bbox_label_t0 = bbox_label_t0) + cls_label, bbox_label, anchor_weight, pos_count = mx.sym.Custom( + op_type='cascade_refine', + stride=int(stride), + network=config.network, + dataset=config.dataset, + prefix=prefix, + cls_label_t0=cls_label_t0, + cls_pred_t0=cls_pred_t0, + cls_pred=rpn_cls_score_reshape, + bbox_pred_t0=bbox_pred_t0, + bbox_label_t0=bbox_label_t0, + cls_label_raw=cls_label_raw, + cas_gt_boxes=gt_boxes) + if stride in config.CASCADE_CLS_STRIDES: + rpn_cls_prob = mx.symbol.SoftmaxOutput( + data=rpn_cls_score_reshape, + label=cls_label, + multi_output=True, + normalization='valid', + use_ignore=True, + ignore_label=-1, + grad_scale=lr_mult, + name='%s_rpn_cls_prob_stride%d_cas%d' % + (prefix, stride, casid)) + ret_group.append(rpn_cls_prob) + ret_group.append(mx.sym.BlockGrad(cls_label)) + if stride in config.CASCADE_BBOX_STRIDES: + bbox_pred = conv_only(body, + '%s_rpn_bbox_pred_stride%d_cas%d' % + (prefix, stride, casid), + bbox_pred_len * num_anchors, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1)) + + rpn_bbox_pred_reshape = mx.symbol.Reshape( + data=bbox_pred, + shape=(0, 0, -1), + name="%s_rpn_bbox_pred_reshape_stride%s_cas%d" % + (prefix, stride, casid)) + _bbox_weight = mx.sym.tile(anchor_weight, + (1, 1, bbox_pred_len)) + _bbox_weight = _bbox_weight.reshape( + (0, -1, A * bbox_pred_len)).transpose((0, 2, 1)) + bbox_weight = _bbox_weight + pos_count = mx.symbol.sum(pos_count) + pos_count = pos_count + 0.01 #avoid zero + #bbox_weight = mx.sym.elemwise_mul(bbox_weight, _bbox_weight, name='%s_bbox_weight_mul_stride%s'%(prefix,stride)) + #bbox loss + bbox_diff = rpn_bbox_pred_reshape - bbox_label + bbox_diff = bbox_diff * bbox_weight + rpn_bbox_loss_ = mx.symbol.smooth_l1( + name='%s_rpn_bbox_loss_stride%d_cas%d' % + (prefix, stride, casid), + scalar=3.0, + data=bbox_diff) + if config.LR_MODE == 0: + rpn_bbox_loss = mx.sym.MakeLoss( + name='%s_rpn_bbox_loss_stride%d_cas%d' % + (prefix, stride, casid), + data=rpn_bbox_loss_, + grad_scale=bbox_lr_mode0) + else: + rpn_bbox_loss_ = mx.symbol.broadcast_div( + rpn_bbox_loss_, pos_count) + rpn_bbox_loss = mx.sym.MakeLoss( + name='%s_rpn_bbox_loss_stride%d_cas%d' % + (prefix, stride, casid), + data=rpn_bbox_loss_, + grad_scale=0.5 * lr_mult) + ret_group.append(rpn_bbox_loss) + ret_group.append(mx.sym.BlockGrad(bbox_weight)) + #bbox_pred = rpn_bbox_pred_reshape + + return ret_group + + +def get_sym_train(sym): + data = mx.symbol.Variable(name="data") + global F1, F2 + F1 = config.HEAD_FILTER_NUM + F2 = F1 + + # shared convolutional layers + conv_fpn_feat = get_sym_conv(data, sym) + ret_group = [] + gt_boxes = None + if config.CASCADE > 0: + gt_boxes = mx.sym.Variable('gt_boxes') + + for stride in config.RPN_FEAT_STRIDE: + ret = get_out(conv_fpn_feat, + 'face', + stride, + config.FACE_LANDMARK, + lr_mult=1.0, + gt_boxes=gt_boxes) + ret_group += ret + + return mx.sym.Group(ret_group) diff --git a/RetinaFace/rcnn/symbol/symbol_common.py.bak b/detection/RetinaFace/rcnn/symbol/symbol_common.py.bak similarity index 100% rename from RetinaFace/rcnn/symbol/symbol_common.py.bak rename to detection/RetinaFace/rcnn/symbol/symbol_common.py.bak diff --git a/detection/RetinaFace/rcnn/symbol/symbol_mnet.py b/detection/RetinaFace/rcnn/symbol/symbol_mnet.py new file mode 100644 index 0000000..86f65a3 --- /dev/null +++ b/detection/RetinaFace/rcnn/symbol/symbol_mnet.py @@ -0,0 +1,834 @@ +import mxnet as mx +import mxnet.ndarray as nd +import mxnet.gluon as gluon +import mxnet.gluon.nn as nn +import mxnet.autograd as ag +import numpy as np +from rcnn.config import config +from rcnn.PY_OP import rpn_fpn_ohem3 +from rcnn.symbol.symbol_common import get_sym_train + + +def conv_only(from_layer, name, num_filter, kernel=(1,1), pad=(0,0), \ + stride=(1,1), bias_wd_mult=0.0, shared_weight=None, shared_bias = None): + if shared_weight is None: + weight = mx.symbol.Variable(name="{}_weight".format(name), + init=mx.init.Normal(0.01), + attr={'__lr_mult__': '1.0'}) + bias = mx.symbol.Variable(name="{}_bias".format(name), + init=mx.init.Constant(0.0), + attr={ + '__lr_mult__': '2.0', + '__wd_mult__': str(bias_wd_mult) + }) + else: + weight = shared_weight + bias = shared_bias + print('reuse shared var in', name) + conv = mx.symbol.Convolution(data=from_layer, kernel=kernel, pad=pad, \ + stride=stride, num_filter=num_filter, name="{}".format(name), weight = weight, bias=bias) + return conv + +def conv_act_layer_dw(from_layer, name, num_filter, kernel=(1,1), pad=(0,0), \ + stride=(1,1), act_type="relu", bias_wd_mult=0.0): + assert kernel[0] == 3 + weight = mx.symbol.Variable(name="{}_weight".format(name), + init=mx.init.Normal(0.01), + attr={'__lr_mult__': '1.0'}) + bias = mx.symbol.Variable(name="{}_bias".format(name), + init=mx.init.Constant(0.0), + attr={ + '__lr_mult__': '2.0', + '__wd_mult__': str(bias_wd_mult) + }) + conv = mx.symbol.Convolution(data=from_layer, kernel=kernel, pad=pad, \ + stride=stride, num_filter=num_filter, num_group=num_filter, name="{}".format(name), weight=weight, bias=bias) + conv = mx.sym.BatchNorm(data=conv, + fix_gamma=False, + eps=2e-5, + momentum=0.9, + name=name + '_bn') + if len(act_type) > 0: + relu = mx.symbol.Activation(data=conv, act_type=act_type, \ + name="{}_{}".format(name, act_type)) + else: + relu = conv + return relu + +def conv_act_layer(from_layer, name, num_filter, kernel=(1,1), pad=(0,0), \ + stride=(1,1), act_type="relu", bias_wd_mult=0.0, separable=False, filter_in = -1): + + separable = False + if separable: + assert kernel[0] == 3 + if not separable: + weight = mx.symbol.Variable(name="{}_weight".format(name), + init=mx.init.Normal(0.01), + attr={'__lr_mult__': '1.0'}) + bias = mx.symbol.Variable(name="{}_bias".format(name), + init=mx.init.Constant(0.0), + attr={ + '__lr_mult__': '2.0', + '__wd_mult__': str(bias_wd_mult) + }) + conv = mx.symbol.Convolution(data=from_layer, kernel=kernel, pad=pad, \ + stride=stride, num_filter=num_filter, name="{}".format(name), weight=weight, bias=bias) + conv = mx.sym.BatchNorm(data=conv, + fix_gamma=False, + eps=2e-5, + momentum=0.9, + name=name + '_bn') + else: + if filter_in < 0: + filter_in = num_filter + conv = mx.symbol.Convolution(data=from_layer, kernel=kernel, pad=pad, \ + stride=stride, num_filter=filter_in, num_group=filter_in, name="{}_sep".format(name)) + conv = mx.sym.BatchNorm(data=conv, + fix_gamma=False, + eps=2e-5, + momentum=0.9, + name=name + '_sep_bn') + conv = mx.symbol.Activation(data=conv, act_type='relu', \ + name="{}_sep_bn_relu".format(name)) + conv = mx.symbol.Convolution(data=conv, kernel=(1,1), pad=(0,0), \ + stride=(1,1), num_filter=num_filter, name="{}".format(name)) + conv = mx.sym.BatchNorm(data=conv, + fix_gamma=False, + eps=2e-5, + momentum=0.9, + name=name + '_bn') + if len(act_type) > 0: + relu = mx.symbol.Activation(data=conv, act_type=act_type, \ + name="{}_{}".format(name, act_type)) + else: + relu = conv + return relu + + +def ssh_context_module(body, num_filter, filter_in, name): + conv_dimred = conv_act_layer(body, + name + '_conv1', + num_filter, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + separable=True, + filter_in=filter_in) + conv5x5 = conv_act_layer(conv_dimred, + name + '_conv2', + num_filter, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='', + separable=True) + conv7x7_1 = conv_act_layer(conv_dimred, + name + '_conv3_1', + num_filter, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + separable=True) + conv7x7 = conv_act_layer(conv7x7_1, + name + '_conv3_2', + num_filter, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='', + separable=True) + return (conv5x5, conv7x7) + + +def ssh_detection_module(body, num_filter, filter_in, name): + conv3x3 = conv_act_layer(body, + name + '_conv1', + num_filter, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='', + separable=True, + filter_in=filter_in) + conv5x5, conv7x7 = ssh_context_module(body, num_filter // 2, filter_in, + name + '_context') + ret = mx.sym.concat(*[conv3x3, conv5x5, conv7x7], + dim=1, + name=name + '_concat') + ret = mx.symbol.Activation(data=ret, + act_type='relu', + name=name + '_concat_relu') + return ret + + +def upsampling(data, num_filter, name): + #ret = mx.symbol.Deconvolution(data=data, num_filter=num_filter, kernel=(4,4), stride=(2, 2), pad=(1,1), + # num_group = num_filter, no_bias = True, attr={'__lr_mult__': '0.0', '__wd_mult__': '0.0'}, + # name=name) + #ret = mx.symbol.Deconvolution(data=data, num_filter=num_filter, kernel=(2,2), stride=(2, 2), pad=(0,0), + # num_group = num_filter, no_bias = True, attr={'__lr_mult__': '0.0', '__wd_mult__': '0.0'}, + # name=name) + ret = mx.symbol.UpSampling(data, + scale=2, + sample_type='nearest', + workspace=512, + name=name, + num_args=1) + return ret + + +def get_mnet_conv(data, sym): + mm = config.MULTIPLIER + all_layers = sym.get_internals() + #print(all_layers) + ##c1 = all_layers['mobilenetv20_features_linearbottleneck6_relu60_relu6_output'] #96 + #c1 = all_layers['mobilenetv20_features_linearbottleneck5_elemwise_add0_output'] # 16 + ##c2 = all_layers['mobilenetv20_features_linearbottleneck13_relu60_relu6_output'] + #c2 = all_layers['mobilenetv20_features_linearbottleneck12_elemwise_add0_output'] # 48 + ##c3 = all_layers['mobilenetv20_features_linearbottleneck16_batchnorm2_fwd_output'] # 160 + #c3 = all_layers['mobilenetv20_features_linearbottleneck13_batchnorm2_fwd_output'] # 80 + #c1_filter = int(32*mm) + #c2_filter = int(96*mm) + #c3_filter = int(160*mm) + + #c1 = all_layers['mobilenet0_relu10_fwd_output'] + #c2 = all_layers['mobilenet0_relu22_fwd_output'] + #c3 = all_layers['mobilenet0_relu26_fwd_output'] + + #c1 = all_layers['conv_6_relu_output'] + #c2 = all_layers['conv_12_relu_output'] + #c3 = all_layers['conv_14_relu_output'] + #c1_filter = int(256*mm) + #c2_filter = int(512*mm) + #c3_filter = int(1024*mm) + + isize = 640 + _, out_shape, _ = all_layers.infer_shape(data=(1, 3, isize, isize)) + last_entry = None + c1 = None + c2 = None + c3 = None + c1_name = None + c2_name = None + c3_name = None + c1_filter = -1 + c2_filter = -1 + c3_filter = -1 + #print(len(all_layers), len(out_shape)) + #print(all_layers.__class__) + outputs = all_layers.list_outputs() + #print(outputs.__class__, len(outputs)) + count = len(outputs) + for i in range(count): + name = outputs[i] + shape = out_shape[i] + if not name.endswith('_output'): + continue + if len(shape) != 4: + continue + #print(name, shape) + if c1 is None and shape[2] == isize // 16: + cname = last_entry[0] + #print('c1', last_entry) + c1 = all_layers[cname] + c1_name = cname + if c2 is None and shape[2] == isize // 32: + cname = last_entry[0] + #print('c2', last_entry) + c2 = all_layers[cname] + c2_name = cname + if shape[2] == isize // 32: + c3 = all_layers[name] + #print('c3', name, shape) + c3_name = name + + last_entry = (name, shape) + print('cnames', c1_name, c2_name, c3_name) + + F1 = int(256 * mm) + F2 = int(128 * mm) + if config.SHARE_WEIGHT_BBOX or config.SHARE_WEIGHT_LANDMARK: + F2 = F1 + _bwm = 1.0 + if config.NET_MODE == 0: + c1_lateral = conv_act_layer(c1, + 'ssh_m1_red_conv', + F2, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + c2_lateral = conv_act_layer(c2, + 'ssh_m2_red_conv', + F2, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + #conv5_128_up = mx.symbol.Deconvolution(data=conv5_128, num_filter=F2, kernel=(4,4), stride=(2, 2), pad=(1,1), + # num_group = F2, no_bias = True, attr={'__lr_mult__': '0.0', '__wd_mult__': '0.0'}, + # name='ssh_m2_red_upsampling') + #c2_up = mx.symbol.UpSampling(c2_lateral, scale=2, sample_type='nearest', workspace=512, name='ssh_m2_red_up', num_args=1) + c2_up = upsampling(c2_lateral, F2, 'ssh_m2_red_upsampling') + #conv4_128 = mx.symbol.Crop(*[conv4_128, conv5_128_up]) + c2_up = mx.symbol.Crop(*[c2_up, c1_lateral]) + + c1 = c1_lateral + c2_up + + c1 = conv_act_layer(c1, + 'ssh_m1_conv', + F2, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + m1 = ssh_detection_module(c1, F2, F2, 'ssh_m1_det') + m2 = ssh_detection_module(c2, F1, c2_filter, 'ssh_m2_det') + m3 = ssh_detection_module(c3, F1, c3_filter, 'ssh_m3_det') + elif config.NET_MODE == 1: + c3_lateral = conv_act_layer(c3, + 'ssh_c3_lateral', + F2, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + #c3_up = mx.symbol.UpSampling(c3_lateral, scale=2, sample_type='nearest', workspace=512, name='ssh_c3_up', num_args=1) + c3_up = upsampling(c3_lateral, F2, 'ssh_c3_upsampling') + c2_lateral = conv_act_layer(c2, + 'ssh_c2_lateral', + F2, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + c3_up = mx.symbol.Crop(*[c3_up, c2_lateral]) + c2 = c2_lateral + c3_up + c2 = conv_act_layer(c2, + 'ssh_c2_aggr', + F2, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + c1_lateral = conv_act_layer(c1, + 'ssh_m1_red_conv', + F2, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + #c2_up = mx.symbol.UpSampling(c2, scale=2, sample_type='nearest', workspace=512, name='ssh_m2_red_up', num_args=1) + c2_up = upsampling(c2, F2, 'ssh_c2_upsampling') + #conv4_128 = mx.symbol.Crop(*[conv4_128, conv5_128_up]) + c2_up = mx.symbol.Crop(*[c2_up, c1_lateral]) + + c1 = c1_lateral + c2_up + + c1 = conv_act_layer(c1, + 'ssh_m1_conv', + F2, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + m1 = ssh_detection_module(c1, F2, F2, 'ssh_m1_det') + m2 = ssh_detection_module(c2, F1, c2_filter, 'ssh_m2_det') + m3 = ssh_detection_module(c3, F1, c3_filter, 'ssh_m3_det') + elif config.NET_MODE == 2: + c3 = conv_act_layer(c3, + 'ssh_c3_lateral', + F2, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + #c3_up = mx.symbol.UpSampling(c3, scale=2, sample_type='nearest', workspace=512, name='ssh_c3_up', num_args=1) + c3_up = upsampling(c3, F2, 'ssh_c3_upsampling') + c2_lateral = conv_act_layer(c2, + 'ssh_c2_lateral', + F2, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + c3_up = mx.symbol.Crop(*[c3_up, c2_lateral]) + c2 = c2_lateral + c3_up + c2 = conv_act_layer(c2, + 'ssh_c2_aggr', + F2, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + c1_lateral = conv_act_layer(c1, + 'ssh_m1_red_conv', + F2, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + #c2_up = mx.symbol.UpSampling(c2, scale=2, sample_type='nearest', workspace=512, name='ssh_m2_red_up', num_args=1) + c2_up = upsampling(c2, F2, 'ssh_c2_upsampling') + #conv4_128 = mx.symbol.Crop(*[conv4_128, conv5_128_up]) + c2_up = mx.symbol.Crop(*[c2_up, c1_lateral]) + c1 = c1_lateral + c2_up + c1 = conv_act_layer(c1, + 'ssh_c1_aggr', + F2, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + + m1 = ssh_detection_module(c1, F2, F2, 'ssh_m1_det') + m2 = ssh_detection_module(c2, F1, c2_filter, 'ssh_m2_det') + m3 = ssh_detection_module(c3, F1, c3_filter, 'ssh_m3_det') + elif config.NET_MODE == 3: + #c3 = conv_act_layer(c3, 'ssh_c3_lateral', + # F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) + c3 = ssh_detection_module(c3, F2 // 2, c3_filter, 'ssh_c3_lateral') + #c3_up = mx.symbol.UpSampling(c3, scale=2, sample_type='nearest', workspace=512, name='ssh_c3_up', num_args=1) + c3_up = upsampling(c3, F2, 'ssh_c3_upsampling') + #c2_lateral = conv_act_layer(c2, 'ssh_c2_lateral', + # F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) + c2_lateral = ssh_detection_module(c2, F2 // 2, c2_filter, + 'ssh_c2_lateral') + c3_up = mx.symbol.Crop(*[c3_up, c2_lateral]) + c2 = c2_lateral + c3_up + c2 = conv_act_layer(c2, + 'ssh_c2_aggr', + F2, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + #c1_lateral = conv_act_layer(c1, 'ssh_m1_red_conv', + # F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) + c1_lateral = ssh_detection_module(c1, F2 // 2, c1_filter, + 'ssh_c1_lateral') + #c2_up = mx.symbol.UpSampling(c2, scale=2, sample_type='nearest', workspace=512, name='ssh_m2_red_up', num_args=1) + c2_up = upsampling(c2, F2, 'ssh_c2_upsampling') + #conv4_128 = mx.symbol.Crop(*[conv4_128, conv5_128_up]) + c2_up = mx.symbol.Crop(*[c2_up, c1_lateral]) + c1 = c1_lateral + c2_up + c1 = conv_act_layer(c1, + 'ssh_c1_aggr', + F2, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + + m1 = ssh_detection_module(c1, F2, F2, 'ssh_m1_det') + m2 = ssh_detection_module(c2, F1, c2_filter, 'ssh_m2_det') + m3 = ssh_detection_module(c3, F1, c3_filter, 'ssh_m3_det') + elif config.NET_MODE == 4: + c3 = conv_act_layer(c3, + 'ssh_c3_lateral', + F2, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + #c3_up = mx.symbol.UpSampling(c3, scale=2, sample_type='nearest', workspace=512, name='ssh_c3_up', num_args=1) + c3_up = upsampling(c3, F2, 'ssh_c3_upsampling') + c2_lateral = conv_act_layer(c2, + 'ssh_c2_lateral', + F2, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + c3_up = mx.symbol.Crop(*[c3_up, c2_lateral]) + c2 = c2_lateral + c3_up + c2 = conv_act_layer(c2, + 'ssh_c2_aggr', + F2, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + c1_lateral = conv_act_layer(c1, + 'ssh_m1_red_conv', + F2, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + #c2_up = mx.symbol.UpSampling(c2, scale=2, sample_type='nearest', workspace=512, name='ssh_m2_red_up', num_args=1) + c2_up = upsampling(c2, F2, 'ssh_c2_upsampling') + #conv4_128 = mx.symbol.Crop(*[conv4_128, conv5_128_up]) + c2_up = mx.symbol.Crop(*[c2_up, c1_lateral]) + c1 = c1_lateral + c2_up + c1 = conv_act_layer(c1, + 'ssh_c1_aggr', + F2, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + + m1 = ssh_detection_module(c1, F2 // 2, F2, 'ssh_m1_det') + m2 = ssh_detection_module(c2, F1 // 2, c2_filter, 'ssh_m2_det') + m3 = ssh_detection_module(c3, F1 // 2, c3_filter, 'ssh_m3_det') + elif config.NET_MODE == 5: + c3 = conv_act_layer_dw(c3, + 'ssh_c3_lateral_m', + F2, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + c3 = conv_act_layer(c3, + 'ssh_c3_lateral', + F2, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + #c3_up = mx.symbol.UpSampling(c3, scale=2, sample_type='nearest', workspace=512, name='ssh_c3_up', num_args=1) + c3_up = upsampling(c3, F2, 'ssh_c3_upsampling') + c2 = conv_act_layer_dw(c2, + 'ssh_c2_lateral_m', + F2, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + c2_lateral = conv_act_layer(c2, + 'ssh_c2_lateral', + F2, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + c3_up = mx.symbol.Crop(*[c3_up, c2_lateral]) + c2 = c2_lateral + c3_up + c2 = conv_act_layer(c2, + 'ssh_c2_aggr', + F2, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + c1 = conv_act_layer_dw(c1, + 'ssh_c1_lateral_m', + F2, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + c1_lateral = conv_act_layer(c1, + 'ssh_m1_red_conv', + F2, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + #c2_up = mx.symbol.UpSampling(c2, scale=2, sample_type='nearest', workspace=512, name='ssh_m2_red_up', num_args=1) + c2_up = upsampling(c2, F2, 'ssh_c2_upsampling') + #conv4_128 = mx.symbol.Crop(*[conv4_128, conv5_128_up]) + c2_up = mx.symbol.Crop(*[c2_up, c1_lateral]) + c1 = c1_lateral + c2_up + c1 = conv_act_layer(c1, + 'ssh_c1_aggr', + F2, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + + m1 = ssh_detection_module(c1, F2, F2, 'ssh_m1_det') + m2 = ssh_detection_module(c2, F1, c2_filter, 'ssh_m2_det') + m3 = ssh_detection_module(c3, F1, c3_filter, 'ssh_m3_det') + + return {8: m1, 16: m2, 32: m3} + + +def get_out(conv_fpn_feat, + prefix, + stride, + landmark=False, + lr_mult=1.0, + shared_vars=None): + A = config.NUM_ANCHORS + bbox_pred_len = 4 + landmark_pred_len = 10 + if config.USE_BLUR: + bbox_pred_len = 5 + if config.USE_OCCLUSION: + landmark_pred_len = 15 + ret_group = [] + num_anchors = config.RPN_ANCHOR_CFG[str(stride)]['NUM_ANCHORS'] + label = mx.symbol.Variable(name='%s_label_stride%d' % (prefix, stride)) + bbox_target = mx.symbol.Variable(name='%s_bbox_target_stride%d' % + (prefix, stride)) + bbox_weight = mx.symbol.Variable(name='%s_bbox_weight_stride%d' % + (prefix, stride)) + if landmark: + landmark_target = mx.symbol.Variable( + name='%s_landmark_target_stride%d' % (prefix, stride)) + landmark_weight = mx.symbol.Variable( + name='%s_landmark_weight_stride%d' % (prefix, stride)) + rpn_relu = conv_fpn_feat[stride] + maxout_stat = 0 + if config.USE_MAXOUT >= 1 and stride == config.RPN_FEAT_STRIDE[-1]: + maxout_stat = 1 + if config.USE_MAXOUT >= 2 and stride != config.RPN_FEAT_STRIDE[-1]: + maxout_stat = 2 + + if maxout_stat == 0: + rpn_cls_score = conv_only(rpn_relu, + '%s_rpn_cls_score_stride%d' % + (prefix, stride), + 2 * num_anchors, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + shared_weight=shared_vars[0][0], + shared_bias=shared_vars[0][1]) + elif maxout_stat == 1: + cls_list = [] + for a in range(num_anchors): + rpn_cls_score_bg = conv_only( + rpn_relu, + '%s_rpn_cls_score_stride%d_anchor%d_bg' % (prefix, stride, a), + 3, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1)) + rpn_cls_score_bg = mx.sym.max(rpn_cls_score_bg, + axis=1, + keepdims=True) + cls_list.append(rpn_cls_score_bg) + rpn_cls_score_fg = conv_only( + rpn_relu, + '%s_rpn_cls_score_stride%d_anchor%d_fg' % (prefix, stride, a), + 1, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1)) + cls_list.append(rpn_cls_score_fg) + rpn_cls_score = mx.sym.concat(*cls_list, + dim=1, + name='%s_rpn_cls_score_stride%d' % + (prefix, stride)) + else: + cls_list = [] + for a in range(num_anchors): + rpn_cls_score_bg = conv_only( + rpn_relu, + '%s_rpn_cls_score_stride%d_anchor%d_bg' % (prefix, stride, a), + 1, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1)) + cls_list.append(rpn_cls_score_bg) + rpn_cls_score_fg = conv_only( + rpn_relu, + '%s_rpn_cls_score_stride%d_anchor%d_fg' % (prefix, stride, a), + 3, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1)) + rpn_cls_score_fg = mx.sym.max(rpn_cls_score_fg, + axis=1, + keepdims=True) + cls_list.append(rpn_cls_score_fg) + rpn_cls_score = mx.sym.concat(*cls_list, + dim=1, + name='%s_rpn_cls_score_stride%d' % + (prefix, stride)) + + rpn_bbox_pred = conv_only(rpn_relu, + '%s_rpn_bbox_pred_stride%d' % (prefix, stride), + bbox_pred_len * num_anchors, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + shared_weight=shared_vars[1][0], + shared_bias=shared_vars[1][1]) + + # prepare rpn data + if not config.FBN: + rpn_cls_score_reshape = mx.symbol.Reshape( + data=rpn_cls_score, + shape=(0, 2, -1), + name="%s_rpn_cls_score_reshape_stride%s" % (prefix, stride)) + else: + rpn_cls_score_reshape = mx.symbol.Reshape( + data=rpn_cls_score, + shape=(0, 2, -1), + name="%s_rpn_cls_score_reshape_stride%s_pre" % (prefix, stride)) + rpn_cls_score_reshape = mx.symbol.BatchNorm( + rpn_cls_score_reshape, + fix_gamma=True, + eps=2e-5, + name="%s_rpn_cls_score_reshape_stride%s" % (prefix, stride)) + + rpn_bbox_pred_reshape = mx.symbol.Reshape( + data=rpn_bbox_pred, + shape=(0, 0, -1), + name="%s_rpn_bbox_pred_reshape_stride%s" % (prefix, stride)) + if landmark: + rpn_landmark_pred = conv_only(rpn_relu, + '%s_rpn_landmark_pred_stride%d' % + (prefix, stride), + landmark_pred_len * num_anchors, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + shared_weight=shared_vars[2][0], + shared_bias=shared_vars[2][1]) + rpn_landmark_pred_reshape = mx.symbol.Reshape( + data=rpn_landmark_pred, + shape=(0, 0, -1), + name="%s_rpn_landmark_pred_reshape_stride%s" % (prefix, stride)) + + if config.TRAIN.RPN_ENABLE_OHEM >= 2: + label, anchor_weight = mx.sym.Custom(op_type='rpn_fpn_ohem3', + stride=int(stride), + network=config.network, + dataset=config.dataset, + prefix=prefix, + cls_score=rpn_cls_score_reshape, + labels=label) + + _bbox_weight = mx.sym.tile(anchor_weight, (1, 1, bbox_pred_len)) + _bbox_weight = _bbox_weight.reshape( + (0, -1, A * bbox_pred_len)).transpose((0, 2, 1)) + bbox_weight = mx.sym.elemwise_mul(bbox_weight, + _bbox_weight, + name='%s_bbox_weight_mul_stride%s' % + (prefix, stride)) + + if landmark: + _landmark_weight = mx.sym.tile(anchor_weight, + (1, 1, landmark_pred_len)) + _landmark_weight = _landmark_weight.reshape( + (0, -1, A * landmark_pred_len)).transpose((0, 2, 1)) + landmark_weight = mx.sym.elemwise_mul( + landmark_weight, + _landmark_weight, + name='%s_landmark_weight_mul_stride%s' % (prefix, stride)) + #if not config.FACE_LANDMARK: + # label, bbox_weight = mx.sym.Custom(op_type='rpn_fpn_ohem', stride=int(stride), cls_score=rpn_cls_score_reshape, bbox_weight = bbox_weight , labels = label) + #else: + # label, bbox_weight, landmark_weight = mx.sym.Custom(op_type='rpn_fpn_ohem2', stride=int(stride), cls_score=rpn_cls_score_reshape, bbox_weight = bbox_weight, landmark_weight=landmark_weight, labels = label) + #cls loss + rpn_cls_prob = mx.symbol.SoftmaxOutput(data=rpn_cls_score_reshape, + label=label, + multi_output=True, + normalization='valid', + use_ignore=True, + ignore_label=-1, + grad_scale=lr_mult, + name='%s_rpn_cls_prob_stride%d' % + (prefix, stride)) + ret_group.append(rpn_cls_prob) + ret_group.append(mx.sym.BlockGrad(label)) + + #bbox loss + bbox_diff = rpn_bbox_pred_reshape - bbox_target + bbox_diff = bbox_diff * bbox_weight + rpn_bbox_loss_ = mx.symbol.smooth_l1(name='%s_rpn_bbox_loss_stride%d_' % + (prefix, stride), + scalar=3.0, + data=bbox_diff) + rpn_bbox_loss = mx.sym.MakeLoss( + name='%s_rpn_bbox_loss_stride%d' % (prefix, stride), + data=rpn_bbox_loss_, + grad_scale=1.0 * lr_mult / (config.TRAIN.RPN_BATCH_SIZE)) + ret_group.append(rpn_bbox_loss) + ret_group.append(mx.sym.BlockGrad(bbox_weight)) + + #landmark loss + if landmark: + landmark_diff = rpn_landmark_pred_reshape - landmark_target + landmark_diff = landmark_diff * landmark_weight + rpn_landmark_loss_ = mx.symbol.smooth_l1( + name='%s_rpn_landmark_loss_stride%d_' % (prefix, stride), + scalar=3.0, + data=landmark_diff) + rpn_landmark_loss = mx.sym.MakeLoss( + name='%s_rpn_landmark_loss_stride%d' % (prefix, stride), + data=rpn_landmark_loss_, + grad_scale=0.5 * lr_mult / (config.TRAIN.RPN_BATCH_SIZE)) + ret_group.append(rpn_landmark_loss) + ret_group.append(mx.sym.BlockGrad(landmark_weight)) + return ret_group + + +def get_mnet_train(sym): + return get_sym_train(sym) + #data = mx.symbol.Variable(name="data") + ## shared convolutional layers + #conv_fpn_feat = get_mnet_conv(data, sym) + #ret_group = [] + #shared_vars = [] + #if config.SHARE_WEIGHT_BBOX: + # assert config.USE_MAXOUT==0 + # _name = 'face_rpn_cls_score_share' + # shared_weight = mx.symbol.Variable(name="{}_weight".format(_name), + # init=mx.init.Normal(0.01), attr={'__lr_mult__': '1.0'}) + # shared_bias = mx.symbol.Variable(name="{}_bias".format(_name), + # init=mx.init.Constant(0.0), attr={'__lr_mult__': '2.0', '__wd_mult__': str(0.0)}) + # shared_vars.append( [shared_weight, shared_bias] ) + # _name = 'face_rpn_bbox_pred_share' + # shared_weight = mx.symbol.Variable(name="{}_weight".format(_name), + # init=mx.init.Normal(0.01), attr={'__lr_mult__': '1.0'}) + # shared_bias = mx.symbol.Variable(name="{}_bias".format(_name), + # init=mx.init.Constant(0.0), attr={'__lr_mult__': '2.0', '__wd_mult__': str(0.0)}) + # shared_vars.append( [shared_weight, shared_bias] ) + #else: + # shared_vars.append( [None, None] ) + # shared_vars.append( [None, None] ) + #if config.SHARE_WEIGHT_LANDMARK: + # _name = 'face_rpn_landmark_pred_share' + # shared_weight = mx.symbol.Variable(name="{}_weight".format(_name), + # init=mx.init.Normal(0.01), attr={'__lr_mult__': '1.0'}) + # shared_bias = mx.symbol.Variable(name="{}_bias".format(_name), + # init=mx.init.Constant(0.0), attr={'__lr_mult__': '2.0', '__wd_mult__': str(0.0)}) + # shared_vars.append( [shared_weight, shared_bias] ) + #else: + # shared_vars.append( [None, None] ) + + #for stride in config.RPN_FEAT_STRIDE: + # ret = get_out(conv_fpn_feat, 'face', stride, config.FACE_LANDMARK, lr_mult=1.0, shared_vars = shared_vars) + # ret_group += ret + # if config.HEAD_BOX: + # ret = get_out(conv_fpn_feat, 'head', stride, False, lr_mult=0.5) + # ret_group += ret + + #return mx.sym.Group(ret_group) diff --git a/RetinaFace/rcnn/symbol/symbol_mnet.py.bak b/detection/RetinaFace/rcnn/symbol/symbol_mnet.py.bak similarity index 100% rename from RetinaFace/rcnn/symbol/symbol_mnet.py.bak rename to detection/RetinaFace/rcnn/symbol/symbol_mnet.py.bak diff --git a/detection/RetinaFace/rcnn/symbol/symbol_resnet.py b/detection/RetinaFace/rcnn/symbol/symbol_resnet.py new file mode 100644 index 0000000..7e6312b --- /dev/null +++ b/detection/RetinaFace/rcnn/symbol/symbol_resnet.py @@ -0,0 +1,827 @@ +import mxnet as mx +import mxnet.ndarray as nd +import mxnet.gluon as gluon +import mxnet.gluon.nn as nn +import mxnet.autograd as ag +import numpy as np +from rcnn.config import config +from rcnn.PY_OP import rpn_fpn_ohem3 +from rcnn.symbol.symbol_common import get_sym_train + +def conv_only(from_layer, name, num_filter, kernel=(1,1), pad=(0,0), \ + stride=(1,1), bias_wd_mult=0.0): + weight = mx.symbol.Variable(name="{}_weight".format(name), + init=mx.init.Normal(0.01), + attr={'__lr_mult__': '1.0'}) + bias = mx.symbol.Variable(name="{}_bias".format(name), + init=mx.init.Constant(0.0), + attr={ + '__lr_mult__': '2.0', + '__wd_mult__': str(bias_wd_mult) + }) + conv = mx.symbol.Convolution(data=from_layer, kernel=kernel, pad=pad, \ + stride=stride, num_filter=num_filter, name="{}".format(name), weight = weight, bias=bias) + return conv + +def conv_act_layer_dw(from_layer, name, num_filter, kernel=(1,1), pad=(0,0), \ + stride=(1,1), act_type="relu", bias_wd_mult=0.0): + assert kernel[0] == 3 + weight = mx.symbol.Variable(name="{}_weight".format(name), + init=mx.init.Normal(0.01), + attr={'__lr_mult__': '1.0'}) + bias = mx.symbol.Variable(name="{}_bias".format(name), + init=mx.init.Constant(0.0), + attr={ + '__lr_mult__': '2.0', + '__wd_mult__': str(bias_wd_mult) + }) + conv = mx.symbol.Convolution(data=from_layer, kernel=kernel, pad=pad, \ + stride=stride, num_filter=num_filter, num_group=num_filter, name="{}".format(name), weight=weight, bias=bias) + conv = mx.sym.BatchNorm(data=conv, + fix_gamma=False, + eps=2e-5, + momentum=0.9, + name=name + '_bn') + if len(act_type) > 0: + relu = mx.symbol.Activation(data=conv, act_type=act_type, \ + name="{}_{}".format(name, act_type)) + else: + relu = conv + return relu + +def conv_act_layer(from_layer, name, num_filter, kernel=(1,1), pad=(0,0), \ + stride=(1,1), act_type="relu", bias_wd_mult=0.0, separable=False, filter_in = -1): + + separable = False + if separable: + assert kernel[0] == 3 + if not separable: + weight = mx.symbol.Variable(name="{}_weight".format(name), + init=mx.init.Normal(0.01), + attr={'__lr_mult__': '1.0'}) + bias = mx.symbol.Variable(name="{}_bias".format(name), + init=mx.init.Constant(0.0), + attr={ + '__lr_mult__': '2.0', + '__wd_mult__': str(bias_wd_mult) + }) + conv = mx.symbol.Convolution(data=from_layer, kernel=kernel, pad=pad, \ + stride=stride, num_filter=num_filter, name="{}".format(name), weight=weight, bias=bias) + conv = mx.sym.BatchNorm(data=conv, + fix_gamma=False, + eps=2e-5, + momentum=0.9, + name=name + '_bn') + else: + if filter_in < 0: + filter_in = num_filter + conv = mx.symbol.Convolution(data=from_layer, kernel=kernel, pad=pad, \ + stride=stride, num_filter=filter_in, num_group=filter_in, name="{}_sep".format(name)) + conv = mx.sym.BatchNorm(data=conv, + fix_gamma=False, + eps=2e-5, + momentum=0.9, + name=name + '_sep_bn') + conv = mx.symbol.Activation(data=conv, act_type='relu', \ + name="{}_sep_bn_relu".format(name)) + conv = mx.symbol.Convolution(data=conv, kernel=(1,1), pad=(0,0), \ + stride=(1,1), num_filter=num_filter, name="{}".format(name)) + conv = mx.sym.BatchNorm(data=conv, + fix_gamma=False, + eps=2e-5, + momentum=0.9, + name=name + '_bn') + if len(act_type) > 0: + relu = mx.symbol.Activation(data=conv, act_type=act_type, \ + name="{}_{}".format(name, act_type)) + else: + relu = conv + return relu + + +def ssh_context_module(body, num_filter, filter_in, name): + conv_dimred = conv_act_layer(body, + name + '_conv1', + num_filter, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + separable=True, + filter_in=filter_in) + conv5x5 = conv_act_layer(conv_dimred, + name + '_conv2', + num_filter, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='', + separable=True) + conv7x7_1 = conv_act_layer(conv_dimred, + name + '_conv3_1', + num_filter, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + separable=True) + conv7x7 = conv_act_layer(conv7x7_1, + name + '_conv3_2', + num_filter, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='', + separable=True) + return (conv5x5, conv7x7) + + +def conv_deformable(net, num_filter, num_group=1, act_type='relu', name=''): + f = num_group * 18 + conv_offset = mx.symbol.Convolution(name=name + '_conv_offset', + data=net, + num_filter=f, + pad=(1, 1), + kernel=(3, 3), + stride=(1, 1)) + net = mx.contrib.symbol.DeformableConvolution( + name=name + "_conv", + data=net, + offset=conv_offset, + num_filter=num_filter, + pad=(1, 1), + kernel=(3, 3), + num_deformable_group=num_group, + stride=(1, 1), + no_bias=False) + net = mx.sym.BatchNorm(data=net, + fix_gamma=False, + eps=2e-5, + momentum=0.9, + name=name + '_bn') + if len(act_type) > 0: + net = mx.symbol.Activation(data=net, + act_type=act_type, + name=name + '_act') + return net + + +def ssh_detection_module(body, num_filter, filter_in, name): + conv3x3 = conv_act_layer(body, + name + '_conv1', + num_filter, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='', + separable=True, + filter_in=filter_in) + conv5x5, conv7x7 = ssh_context_module(body, num_filter // 2, filter_in, + name + '_context') + ret = mx.sym.concat(*[conv3x3, conv5x5, conv7x7], + dim=1, + name=name + '_concat') + ret = mx.symbol.Activation(data=ret, + act_type='relu', + name=name + '_concat_relu') + if config.USE_DCN == 1: + ret = conv_deformable(ret, + num_filter=num_filter * 2, + name=name + '_concat_dcn') + elif config.USE_DCN == 2: + ret = conv_deformable2(ret, + num_filter=num_filter * 2, + name=name + '_concat_dcn') + return ret + + +def get_resnet_conv(data, sym): + all_layers = sym.get_internals() + isize = 640 + _, out_shape, _ = all_layers.infer_shape(data=(1, 3, isize, isize)) + last_entry = None + c1 = None + c2 = None + c3 = None + #print(len(all_layers), len(out_shape)) + #print(all_layers.__class__) + outputs = all_layers.list_outputs() + #print(outputs.__class__, len(outputs)) + count = len(outputs) + for i in range(count): + name = outputs[i] + shape = out_shape[i] + if not name.endswith('_output'): + continue + if len(shape) != 4: + continue + print(name, shape) + if c1 is None and shape[2] == isize // 16: + cname = last_entry[0] + print('c1', last_entry) + c1 = all_layers[cname] + if c2 is None and shape[2] == isize // 32: + cname = last_entry[0] + print('c2', last_entry) + c2 = all_layers[cname] + if shape[2] == isize // 32: + c3 = all_layers[name] + print('c3', name, shape) + + last_entry = (name, shape) + + c1_filter = -1 + c2_filter = -1 + c3_filter = -1 + + F1 = 256 + F2 = 256 + _bwm = 1.0 + if config.NET_MODE == 0: + c1_lateral = conv_act_layer(c1, + 'ssh_m1_red_conv', + F2, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + c2_lateral = conv_act_layer(c2, + 'ssh_m2_red_conv', + F2, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + #conv5_128_up = mx.symbol.Deconvolution(data=conv5_128, num_filter=F2, kernel=(4,4), stride=(2, 2), pad=(1,1), + # num_group = F2, no_bias = True, attr={'__lr_mult__': '0.0', '__wd_mult__': '0.0'}, + # name='ssh_m2_red_upsampling') + c2_up = mx.symbol.UpSampling(c2_lateral, + scale=2, + sample_type='nearest', + workspace=512, + name='ssh_m2_red_up', + num_args=1) + #conv4_128 = mx.symbol.Crop(*[conv4_128, conv5_128_up]) + c2_up = mx.symbol.Crop(*[c2_up, c1_lateral]) + + c1 = c1_lateral + c2_up + + c1 = conv_act_layer(c1, + 'ssh_m1_conv', + F2, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + m1 = ssh_detection_module(c1, F2, F2, 'ssh_m1_det') + m2 = ssh_detection_module(c2, F1, c2_filter, 'ssh_m2_det') + m3 = ssh_detection_module(c3, F1, c3_filter, 'ssh_m3_det') + elif config.NET_MODE == 1: + c3_lateral = conv_act_layer(c3, + 'ssh_c3_lateral', + F2, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + c3_up = mx.symbol.UpSampling(c3_lateral, + scale=2, + sample_type='nearest', + workspace=512, + name='ssh_c3_up', + num_args=1) + c2_lateral = conv_act_layer(c2, + 'ssh_c2_lateral', + F2, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + c3_up = mx.symbol.Crop(*[c3_up, c2_lateral]) + c2 = c2_lateral + c3_up + c2 = conv_act_layer(c2, + 'ssh_c2_aggr', + F2, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + c1_lateral = conv_act_layer(c1, + 'ssh_m1_red_conv', + F2, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + c2_up = mx.symbol.UpSampling(c2, + scale=2, + sample_type='nearest', + workspace=512, + name='ssh_m2_red_up', + num_args=1) + #conv4_128 = mx.symbol.Crop(*[conv4_128, conv5_128_up]) + c2_up = mx.symbol.Crop(*[c2_up, c1_lateral]) + + c1 = c1_lateral + c2_up + + c1 = conv_act_layer(c1, + 'ssh_m1_conv', + F2, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + m1 = ssh_detection_module(c1, F2, F2, 'ssh_m1_det') + m2 = ssh_detection_module(c2, F1, c2_filter, 'ssh_m2_det') + m3 = ssh_detection_module(c3, F1, c3_filter, 'ssh_m3_det') + elif config.NET_MODE == 2: + n1 = ssh_detection_module(c1, F2, F2, 'ssh_n1_det') + n2 = ssh_detection_module(c2, F1, c2_filter, 'ssh_n2_det') + n3 = ssh_detection_module(c3, F1, c3_filter, 'ssh_n3_det') + c3 = conv_act_layer(c3, + 'ssh_c3_lateral', + F2, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + c3_up = mx.symbol.UpSampling(c3, + scale=2, + sample_type='nearest', + workspace=512, + name='ssh_c3_up', + num_args=1) + c2_lateral = conv_act_layer(c2, + 'ssh_c2_lateral', + F2, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + c3_up = mx.symbol.Crop(*[c3_up, c2_lateral]) + c2 = c2_lateral + c3_up + c2 = conv_act_layer(c2, + 'ssh_c2_aggr', + F2, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + c1_lateral = conv_act_layer(c1, + 'ssh_m1_red_conv', + F2, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + c2_up = mx.symbol.UpSampling(c2, + scale=2, + sample_type='nearest', + workspace=512, + name='ssh_m2_red_up', + num_args=1) + #conv4_128 = mx.symbol.Crop(*[conv4_128, conv5_128_up]) + c2_up = mx.symbol.Crop(*[c2_up, c1_lateral]) + c1 = c1_lateral + c2_up + c1 = conv_act_layer(c1, + 'ssh_c1_aggr', + F2, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + + m1 = ssh_detection_module(c1, F2, F2, 'ssh_m1_det') + m2 = ssh_detection_module(c2, F1, c2_filter, 'ssh_m2_det') + m3 = ssh_detection_module(c3, F1, c3_filter, 'ssh_m3_det') + elif config.NET_MODE == 3: + #c3 = conv_act_layer(c3, 'ssh_c3_lateral', + # F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) + c3 = ssh_detection_module(c3, F2 // 2, c3_filter, 'ssh_c3_lateral') + c3_up = mx.symbol.UpSampling(c3, + scale=2, + sample_type='nearest', + workspace=512, + name='ssh_c3_up', + num_args=1) + #c2_lateral = conv_act_layer(c2, 'ssh_c2_lateral', + # F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) + c2_lateral = ssh_detection_module(c2, F2 // 2, c2_filter, + 'ssh_c2_lateral') + c3_up = mx.symbol.Crop(*[c3_up, c2_lateral]) + c2 = c2_lateral + c3_up + c2 = conv_act_layer(c2, + 'ssh_c2_aggr', + F2, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + #c1_lateral = conv_act_layer(c1, 'ssh_m1_red_conv', + # F2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu', bias_wd_mult=_bwm) + c1_lateral = ssh_detection_module(c1, F2 // 2, c1_filter, + 'ssh_c1_lateral') + c2_up = mx.symbol.UpSampling(c2, + scale=2, + sample_type='nearest', + workspace=512, + name='ssh_m2_red_up', + num_args=1) + #conv4_128 = mx.symbol.Crop(*[conv4_128, conv5_128_up]) + c2_up = mx.symbol.Crop(*[c2_up, c1_lateral]) + c1 = c1_lateral + c2_up + c1 = conv_act_layer(c1, + 'ssh_c1_aggr', + F2, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + + m1 = ssh_detection_module(c1, F2, F2, 'ssh_m1_det') + m2 = ssh_detection_module(c2, F1, c2_filter, 'ssh_m2_det') + m3 = ssh_detection_module(c3, F1, c3_filter, 'ssh_m3_det') + elif config.NET_MODE == 4: + c3 = conv_act_layer(c3, + 'ssh_c3_lateral', + F2, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + c3_up = mx.symbol.UpSampling(c3, + scale=2, + sample_type='nearest', + workspace=512, + name='ssh_c3_up', + num_args=1) + c2_lateral = conv_act_layer(c2, + 'ssh_c2_lateral', + F2, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + c3_up = mx.symbol.Crop(*[c3_up, c2_lateral]) + c2 = c2_lateral + c3_up + c2 = conv_act_layer(c2, + 'ssh_c2_aggr', + F2, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + c1_lateral = conv_act_layer(c1, + 'ssh_m1_red_conv', + F2, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + c2_up = mx.symbol.UpSampling(c2, + scale=2, + sample_type='nearest', + workspace=512, + name='ssh_m2_red_up', + num_args=1) + #conv4_128 = mx.symbol.Crop(*[conv4_128, conv5_128_up]) + c2_up = mx.symbol.Crop(*[c2_up, c1_lateral]) + c1 = c1_lateral + c2_up + c1 = conv_act_layer(c1, + 'ssh_c1_aggr', + F2, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + + m1 = ssh_detection_module(c1, F2 // 2, F2, 'ssh_m1_det') + m2 = ssh_detection_module(c2, F1 // 2, c2_filter, 'ssh_m2_det') + m3 = ssh_detection_module(c3, F1 // 2, c3_filter, 'ssh_m3_det') + elif config.NET_MODE == 5: + c3 = conv_act_layer_dw(c3, + 'ssh_c3_lateral_m', + F2, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + c3 = conv_act_layer(c3, + 'ssh_c3_lateral', + F2, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + c3_up = mx.symbol.UpSampling(c3, + scale=2, + sample_type='nearest', + workspace=512, + name='ssh_c3_up', + num_args=1) + c2 = conv_act_layer_dw(c2, + 'ssh_c2_lateral_m', + F2, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + c2_lateral = conv_act_layer(c2, + 'ssh_c2_lateral', + F2, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + c3_up = mx.symbol.Crop(*[c3_up, c2_lateral]) + c2 = c2_lateral + c3_up + c2 = conv_act_layer(c2, + 'ssh_c2_aggr', + F2, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + c1 = conv_act_layer_dw(c1, + 'ssh_c1_lateral_m', + F2, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + c1_lateral = conv_act_layer(c1, + 'ssh_m1_red_conv', + F2, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + c2_up = mx.symbol.UpSampling(c2, + scale=2, + sample_type='nearest', + workspace=512, + name='ssh_m2_red_up', + num_args=1) + #conv4_128 = mx.symbol.Crop(*[conv4_128, conv5_128_up]) + c2_up = mx.symbol.Crop(*[c2_up, c1_lateral]) + c1 = c1_lateral + c2_up + c1 = conv_act_layer(c1, + 'ssh_c1_aggr', + F2, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + + m1 = ssh_detection_module(c1, F2, F2, 'ssh_m1_det') + m2 = ssh_detection_module(c2, F1, c2_filter, 'ssh_m2_det') + m3 = ssh_detection_module(c3, F1, c3_filter, 'ssh_m3_det') + + return {8: m1, 16: m2, 32: m3}, {8: n1, 16: n2, 32: n3} + + +def get_out(conv_fpn_feat, prefix, stride, landmark=False, lr_mult=1.0): + A = config.NUM_ANCHORS + bbox_pred_len = 4 + landmark_pred_len = 10 + if config.USE_BLUR: + bbox_pred_len = 5 + if config.USE_OCCLUSION: + landmark_pred_len = 15 + ret_group = [] + num_anchors = config.RPN_ANCHOR_CFG[str(stride)]['NUM_ANCHORS'] + label = mx.symbol.Variable(name='%s_label_stride%d' % (prefix, stride)) + bbox_target = mx.symbol.Variable(name='%s_bbox_target_stride%d' % + (prefix, stride)) + bbox_weight = mx.symbol.Variable(name='%s_bbox_weight_stride%d' % + (prefix, stride)) + if landmark: + landmark_target = mx.symbol.Variable( + name='%s_landmark_target_stride%d' % (prefix, stride)) + landmark_weight = mx.symbol.Variable( + name='%s_landmark_weight_stride%d' % (prefix, stride)) + rpn_relu = conv_fpn_feat[stride] + maxout_stat = 0 + if config.USE_MAXOUT >= 1 and stride == config.RPN_FEAT_STRIDE[-1]: + maxout_stat = 1 + if config.USE_MAXOUT >= 2 and stride != config.RPN_FEAT_STRIDE[-1]: + maxout_stat = 2 + + if maxout_stat == 0: + rpn_cls_score = conv_only(rpn_relu, + '%s_rpn_cls_score_stride%d' % + (prefix, stride), + 2 * num_anchors, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1)) + elif maxout_stat == 1: + cls_list = [] + for a in range(num_anchors): + rpn_cls_score_bg = conv_only( + rpn_relu, + '%s_rpn_cls_score_stride%d_anchor%d_bg' % (prefix, stride, a), + 3, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1)) + rpn_cls_score_bg = mx.sym.max(rpn_cls_score_bg, + axis=1, + keepdims=True) + cls_list.append(rpn_cls_score_bg) + rpn_cls_score_fg = conv_only( + rpn_relu, + '%s_rpn_cls_score_stride%d_anchor%d_fg' % (prefix, stride, a), + 1, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1)) + cls_list.append(rpn_cls_score_fg) + rpn_cls_score = mx.sym.concat(*cls_list, + dim=1, + name='%s_rpn_cls_score_stride%d' % + (prefix, stride)) + else: + cls_list = [] + for a in range(num_anchors): + rpn_cls_score_bg = conv_only( + rpn_relu, + '%s_rpn_cls_score_stride%d_anchor%d_bg' % (prefix, stride, a), + 1, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1)) + cls_list.append(rpn_cls_score_bg) + rpn_cls_score_fg = conv_only( + rpn_relu, + '%s_rpn_cls_score_stride%d_anchor%d_fg' % (prefix, stride, a), + 3, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1)) + rpn_cls_score_fg = mx.sym.max(rpn_cls_score_fg, + axis=1, + keepdims=True) + cls_list.append(rpn_cls_score_fg) + rpn_cls_score = mx.sym.concat(*cls_list, + dim=1, + name='%s_rpn_cls_score_stride%d' % + (prefix, stride)) + + rpn_bbox_pred = conv_only(rpn_relu, + '%s_rpn_bbox_pred_stride%d' % (prefix, stride), + bbox_pred_len * num_anchors, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1)) + + # prepare rpn data + if not config.FBN: + rpn_cls_score_reshape = mx.symbol.Reshape( + data=rpn_cls_score, + shape=(0, 2, -1), + name="%s_rpn_cls_score_reshape_stride%s" % (prefix, stride)) + else: + rpn_cls_score_reshape = mx.symbol.Reshape( + data=rpn_cls_score, + shape=(0, 2, -1), + name="%s_rpn_cls_score_reshape_stride%s_pre" % (prefix, stride)) + rpn_cls_score_reshape = mx.symbol.BatchNorm( + rpn_cls_score_reshape, + fix_gamma=True, + eps=2e-5, + name="%s_rpn_cls_score_reshape_stride%s" % (prefix, stride)) + + rpn_bbox_pred_reshape = mx.symbol.Reshape( + data=rpn_bbox_pred, + shape=(0, 0, -1), + name="%s_rpn_bbox_pred_reshape_stride%s" % (prefix, stride)) + if landmark: + rpn_landmark_pred = conv_only(rpn_relu, + '%s_rpn_landmark_pred_stride%d' % + (prefix, stride), + landmark_pred_len * num_anchors, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1)) + rpn_landmark_pred_reshape = mx.symbol.Reshape( + data=rpn_landmark_pred, + shape=(0, 0, -1), + name="%s_rpn_landmark_pred_reshape_stride%s" % (prefix, stride)) + + if config.TRAIN.RPN_ENABLE_OHEM >= 2: + label, anchor_weight = mx.sym.Custom(op_type='rpn_fpn_ohem3', + stride=int(stride), + network=config.network, + dataset=config.dataset, + prefix=prefix, + cls_score=rpn_cls_score_reshape, + labels=label) + + _bbox_weight = mx.sym.tile(anchor_weight, (1, 1, bbox_pred_len)) + _bbox_weight = _bbox_weight.reshape( + (0, -1, A * bbox_pred_len)).transpose((0, 2, 1)) + bbox_weight = mx.sym.elemwise_mul(bbox_weight, + _bbox_weight, + name='%s_bbox_weight_mul_stride%s' % + (prefix, stride)) + + if landmark: + _landmark_weight = mx.sym.tile(anchor_weight, + (1, 1, landmark_pred_len)) + _landmark_weight = _landmark_weight.reshape( + (0, -1, A * landmark_pred_len)).transpose((0, 2, 1)) + landmark_weight = mx.sym.elemwise_mul( + landmark_weight, + _landmark_weight, + name='%s_landmark_weight_mul_stride%s' % (prefix, stride)) + #if not config.FACE_LANDMARK: + # label, bbox_weight = mx.sym.Custom(op_type='rpn_fpn_ohem', stride=int(stride), cls_score=rpn_cls_score_reshape, bbox_weight = bbox_weight , labels = label) + #else: + # label, bbox_weight, landmark_weight = mx.sym.Custom(op_type='rpn_fpn_ohem2', stride=int(stride), cls_score=rpn_cls_score_reshape, bbox_weight = bbox_weight, landmark_weight=landmark_weight, labels = label) + #cls loss + rpn_cls_prob = mx.symbol.SoftmaxOutput(data=rpn_cls_score_reshape, + label=label, + multi_output=True, + normalization='valid', + use_ignore=True, + ignore_label=-1, + grad_scale=lr_mult, + name='%s_rpn_cls_prob_stride%d' % + (prefix, stride)) + ret_group.append(rpn_cls_prob) + ret_group.append(mx.sym.BlockGrad(label)) + + #bbox loss + bbox_diff = rpn_bbox_pred_reshape - bbox_target + bbox_diff = bbox_diff * bbox_weight + rpn_bbox_loss_ = mx.symbol.smooth_l1(name='%s_rpn_bbox_loss_stride%d_' % + (prefix, stride), + scalar=3.0, + data=bbox_diff) + rpn_bbox_loss = mx.sym.MakeLoss( + name='%s_rpn_bbox_loss_stride%d' % (prefix, stride), + data=rpn_bbox_loss_, + grad_scale=1.0 * lr_mult / (config.TRAIN.RPN_BATCH_SIZE)) + ret_group.append(rpn_bbox_loss) + ret_group.append(mx.sym.BlockGrad(bbox_weight)) + + #landmark loss + if landmark: + landmark_diff = rpn_landmark_pred_reshape - landmark_target + landmark_diff = landmark_diff * landmark_weight + rpn_landmark_loss_ = mx.symbol.smooth_l1( + name='%s_rpn_landmark_loss_stride%d_' % (prefix, stride), + scalar=3.0, + data=landmark_diff) + rpn_landmark_loss = mx.sym.MakeLoss( + name='%s_rpn_landmark_loss_stride%d' % (prefix, stride), + data=rpn_landmark_loss_, + grad_scale=0.5 * lr_mult / (config.TRAIN.RPN_BATCH_SIZE)) + ret_group.append(rpn_landmark_loss) + ret_group.append(mx.sym.BlockGrad(landmark_weight)) + return ret_group + + +def get_resnet_train(sym): + return get_sym_train(sym) + #data = mx.symbol.Variable(name="data") + ## shared convolutional layers + #conv_fpn_feat, conv_fpn_feat2 = get_resnet_conv(data, sym) + #ret_group = [] + #for stride in config.RPN_FEAT_STRIDE: + # ret = get_out(conv_fpn_feat, 'face', stride, config.FACE_LANDMARK, lr_mult=1.0) + # ret_group += ret + # if config.HEAD_BOX: + # ret = get_out(conv_fpn_feat2, 'head', stride, False, lr_mult=1.0) + # ret_group += ret + + #return mx.sym.Group(ret_group) diff --git a/detection/RetinaFace/rcnn/symbol/symbol_ssh.py b/detection/RetinaFace/rcnn/symbol/symbol_ssh.py new file mode 100644 index 0000000..f8be30d --- /dev/null +++ b/detection/RetinaFace/rcnn/symbol/symbol_ssh.py @@ -0,0 +1,725 @@ +import mxnet as mx +import numpy as np +from rcnn.config import config +from rcnn.PY_OP import rpn_fpn_ohem3 +FPN = False +USE_DCN = False + +def conv_act_layer(from_layer, name, num_filter, kernel=(1,1), pad=(0,0), \ + stride=(1,1), act_type="relu", bias_wd_mult=0.0, dcn=False): + + weight = mx.symbol.Variable(name="{}_weight".format(name), + init=mx.init.Normal(0.01), + attr={'__lr_mult__': '1.0'}) + bias = mx.symbol.Variable(name="{}_bias".format(name), + init=mx.init.Constant(0.0), + attr={ + '__lr_mult__': '2.0', + '__wd_mult__': str(bias_wd_mult) + }) + if not dcn: + conv = mx.symbol.Convolution(data=from_layer, kernel=kernel, pad=pad, \ + stride=stride, num_filter=num_filter, name="{}".format(name), weight = weight, bias=bias) + else: + assert kernel[0] == 3 and kernel[1] == 3 + num_group = 1 + f = num_group * 18 + offset_weight = mx.symbol.Variable( + name="{}_offset_weight".format(name), + init=mx.init.Constant(0.0), + attr={'__lr_mult__': '1.0'}) + offset_bias = mx.symbol.Variable(name="{}_offset_bias".format(name), + init=mx.init.Constant(0.0), + attr={ + '__lr_mult__': '2.0', + '__wd_mult__': str(bias_wd_mult) + }) + conv_offset = mx.symbol.Convolution(name=name + '_offset', + data=from_layer, + weight=offset_weight, + bias=offset_bias, + num_filter=f, + pad=(1, 1), + kernel=(3, 3), + stride=(1, 1)) + conv = mx.contrib.symbol.DeformableConvolution( + name=name, + data=from_layer, + offset=conv_offset, + weight=weight, + bias=bias, + num_filter=num_filter, + pad=(1, 1), + kernel=(3, 3), + num_deformable_group=num_group, + stride=(1, 1), + no_bias=False) + if len(act_type) > 0: + relu = mx.symbol.Activation(data=conv, act_type=act_type, \ + name="{}_{}".format(name, act_type)) + else: + relu = conv + return relu + + +def ssh_context_module(body, num_filters, name): + conv_dimred = conv_act_layer(body, + name + '_conv1', + num_filters, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + dcn=False) + conv5x5 = conv_act_layer(conv_dimred, + name + '_conv2', + num_filters, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='', + dcn=USE_DCN) + conv7x7_1 = conv_act_layer(conv_dimred, + name + '_conv3_1', + num_filters, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + dcn=False) + conv7x7 = conv_act_layer(conv7x7_1, + name + '_conv3_2', + num_filters, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='', + dcn=USE_DCN) + return (conv5x5, conv7x7) + + +def ssh_detection_module(body, num_filters, name): + conv3x3 = conv_act_layer(body, + name + '_conv1', + num_filters, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='', + dcn=USE_DCN) + conv5x5, conv7x7 = ssh_context_module(body, num_filters // 2, + name + '_context') + ret = mx.sym.concat(*[conv3x3, conv5x5, conv7x7], + dim=1, + name=name + '_concat') + ret = mx.symbol.Activation(data=ret, + act_type='relu', + name=name + '_concat_relu') + return ret + + +def conv_bn(input, filter, ksize, stride, padding, act_type='relu', name=''): + conv = mx.symbol.Convolution(data=input, kernel=(ksize,ksize), pad=(padding,padding), \ + stride=(stride,stride), num_filter=filter, name=name+"_conv") + ret = mx.sym.BatchNorm(data=conv, + fix_gamma=False, + eps=2e-5, + momentum=0.9, + name=name + '_bn') + if act_type is not None: + ret = mx.symbol.Activation(data=ret, act_type=act_type, \ + name="{}_{}".format(name, act_type)) + return ret + + +def cpm(input, name): + # residual + branch1 = conv_bn(input, + 1024, + 1, + 1, + 0, + act_type=None, + name=name + "_branch1") + branch2a = conv_bn(input, + 256, + 1, + 1, + 0, + act_type='relu', + name=name + "_branch2a") + branch2b = conv_bn(branch2a, + 256, + 3, + 1, + 1, + act_type='relu', + name=name + "_branch2b") + branch2c = conv_bn(branch2b, + 1024, + 1, + 1, + 0, + act_type=None, + name=name + "_branch2c") + sum = branch1 + branch2c + rescomb = mx.symbol.Activation(data=sum, + act_type='relu', + name="%s_relu2" % (name)) + + ssh_out = ssh_detection_module(rescomb, 256, name=name + "_ssh") + return ssh_out + + +def get_feat_down(conv_feat): + #P5 = mx.symbol.Convolution(data=conv_feat[0], kernel=(1, 1), num_filter=256, name="P5_lateral") + P5 = conv_act_layer(conv_feat[0], + 'P5_lateral', + 256, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='relu') + + # P5 2x upsampling + C4 = P4 + P5_up = mx.symbol.UpSampling(P5, + scale=2, + sample_type='nearest', + workspace=512, + name='P5_upsampling', + num_args=1) + #P4_la = mx.symbol.Convolution(data=conv_feat[1], kernel=(1, 1), num_filter=256, name="P4_lateral") + P4_la = conv_act_layer(conv_feat[1], + 'P4_lateral', + 256, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='relu') + P5_clip = mx.symbol.Crop(*[P5_up, P4_la], name="P4_clip") + P4 = mx.sym.ElementWiseSum(*[P5_clip, P4_la], name="P4_sum") + #P4 = mx.symbol.Convolution(data=P4, kernel=(3, 3), pad=(1, 1), num_filter=256, name="P4_aggregate") + P4 = conv_act_layer(P4, + 'P4_aggregate', + 256, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu') + + # P4 2x upsampling + C3 = P3 + P4_up = mx.symbol.UpSampling(P4, + scale=2, + sample_type='nearest', + workspace=512, + name='P4_upsampling', + num_args=1) + #P3_la = mx.symbol.Convolution(data=conv_feat[2], kernel=(1, 1), num_filter=256, name="P3_lateral") + P3_la = conv_act_layer(conv_feat[2], + 'P3_lateral', + 256, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='relu') + P4_clip = mx.symbol.Crop(*[P4_up, P3_la], name="P3_clip") + P3 = mx.sym.ElementWiseSum(*[P4_clip, P3_la], name="P3_sum") + #P3 = mx.symbol.Convolution(data=P3, kernel=(3, 3), pad=(1, 1), num_filter=256, name="P3_aggregate") + P3 = conv_act_layer(P3, + 'P3_aggregate', + 256, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu') + + return P3, P4, P5 + + +def get_ssh_conv(data): + """ + shared convolutional layers + :param data: Symbol + :return: Symbol + """ + # group 1 + #conv1_1 = mx.symbol.Convolution( + # data=data, kernel=(3, 3), pad=(1, 1), num_filter=64, workspace=2048, name="conv1_1") + #relu1_1 = mx.symbol.Activation(data=conv1_1, act_type="relu", name="relu1_1") + relu1_1 = conv_act_layer(data, + 'conv1_1', + 64, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu') + #conv1_2 = mx.symbol.Convolution( + # data=relu1_1, kernel=(3, 3), pad=(1, 1), num_filter=64, workspace=2048, name="conv1_2") + #relu1_2 = mx.symbol.Activation(data=conv1_2, act_type="relu", name="relu1_2") + relu1_2 = conv_act_layer(relu1_1, + 'conv1_2', + 64, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu') + pool1 = mx.symbol.Pooling(data=relu1_2, + pool_type="max", + kernel=(2, 2), + stride=(2, 2), + name="pool1") + # group 2 + #conv2_1 = mx.symbol.Convolution( + # data=pool1, kernel=(3, 3), pad=(1, 1), num_filter=128, workspace=2048, name="conv2_1") + #relu2_1 = mx.symbol.Activation(data=conv2_1, act_type="relu", name="relu2_1") + relu2_1 = conv_act_layer(pool1, + 'conv2_1', + 128, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu') + #conv2_2 = mx.symbol.Convolution( + # data=relu2_1, kernel=(3, 3), pad=(1, 1), num_filter=128, workspace=2048, name="conv2_2") + #relu2_2 = mx.symbol.Activation(data=conv2_2, act_type="relu", name="relu2_2") + relu2_2 = conv_act_layer(relu2_1, + 'conv2_2', + 128, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu') + pool2 = mx.symbol.Pooling(data=relu2_2, + pool_type="max", + kernel=(2, 2), + stride=(2, 2), + name="pool2") + # group 3 + #conv3_1 = mx.symbol.Convolution( + # data=pool2, kernel=(3, 3), pad=(1, 1), num_filter=256, workspace=2048, name="conv3_1") + #relu3_1 = mx.symbol.Activation(data=conv3_1, act_type="relu", name="relu3_1") + relu3_1 = conv_act_layer(pool2, + 'conv3_1', + 256, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu') + #conv3_2 = mx.symbol.Convolution( + # data=relu3_1, kernel=(3, 3), pad=(1, 1), num_filter=256, workspace=2048, name="conv3_2") + #relu3_2 = mx.symbol.Activation(data=conv3_2, act_type="relu", name="relu3_2") + relu3_2 = conv_act_layer(relu3_1, + 'conv3_2', + 256, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu') + #conv3_3 = mx.symbol.Convolution( + # data=relu3_2, kernel=(3, 3), pad=(1, 1), num_filter=256, workspace=2048, name="conv3_3") + #relu3_3 = mx.symbol.Activation(data=conv3_3, act_type="relu", name="relu3_3") + relu3_3 = conv_act_layer(relu3_2, + 'conv3_3', + 256, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu') + pool3 = mx.symbol.Pooling(data=relu3_3, + pool_type="max", + kernel=(2, 2), + stride=(2, 2), + name="pool3") + # group 4 + #conv4_1 = mx.symbol.Convolution( + # data=pool3, kernel=(3, 3), pad=(1, 1), num_filter=512, workspace=2048, name="conv4_1") + #relu4_1 = mx.symbol.Activation(data=conv4_1, act_type="relu", name="relu4_1") + relu4_1 = conv_act_layer(pool3, + 'conv4_1', + 512, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu') + #conv4_2 = mx.symbol.Convolution( + # data=relu4_1, kernel=(3, 3), pad=(1, 1), num_filter=512, workspace=2048, name="conv4_2") + #relu4_2 = mx.symbol.Activation(data=conv4_2, act_type="relu", name="relu4_2") + relu4_2 = conv_act_layer(relu4_1, + 'conv4_2', + 512, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu') + #conv4_3 = mx.symbol.Convolution( + # data=relu4_2, kernel=(3, 3), pad=(1, 1), num_filter=512, workspace=2048, name="conv4_3") + #relu4_3 = mx.symbol.Activation(data=conv4_3, act_type="relu", name="relu4_3") + relu4_3 = conv_act_layer(relu4_2, + 'conv4_3', + 512, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu') + pool4 = mx.symbol.Pooling(data=relu4_3, + pool_type="max", + kernel=(2, 2), + stride=(2, 2), + name="pool4") + # group 5 + #conv5_1 = mx.symbol.Convolution( + # data=pool4, kernel=(3, 3), pad=(1, 1), num_filter=512, workspace=2048, name="conv5_1") + #relu5_1 = mx.symbol.Activation(data=conv5_1, act_type="relu", name="relu5_1") + relu5_1 = conv_act_layer(pool4, + 'conv5_1', + 512, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu') + #conv5_2 = mx.symbol.Convolution( + # data=relu5_1, kernel=(3, 3), pad=(1, 1), num_filter=512, workspace=2048, name="conv5_2") + #relu5_2 = mx.symbol.Activation(data=conv5_2, act_type="relu", name="relu5_2") + relu5_2 = conv_act_layer(relu5_1, + 'conv5_2', + 512, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu') + #conv5_3 = mx.symbol.Convolution( + # data=relu5_2, kernel=(3, 3), pad=(1, 1), num_filter=512, workspace=2048, name="conv5_3") + #relu5_3 = mx.symbol.Activation(data=conv5_3, act_type="relu", name="relu5_3") + relu5_3 = conv_act_layer(relu5_2, + 'conv5_3', + 512, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu') + m3_pool = mx.sym.Pooling(data=relu5_3, + kernel=(2, 2), + stride=(2, 2), + pad=(0, 0), + pool_type='max') + if config.SSH_MODE <= 5: + #if FPN: + # relu4_3, relu5_3, m3_pool = get_feat_down([m3_pool, relu5_3, relu4_3]) + + F1 = 256 + F2 = 128 + if config.SSH_MODE == 1: + F2 = 256 + _bwm = 1.0 + conv4_128 = conv_act_layer(relu4_3, + 'ssh_m1_red_conv', + F2, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + conv5_128 = conv_act_layer(relu5_3, + 'ssh_m2_red_conv', + F2, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + conv5_128_up = mx.symbol.Deconvolution(data=conv5_128, + num_filter=F2, + kernel=(4, 4), + stride=(2, 2), + pad=(1, 1), + num_group=F2, + no_bias=True, + attr={ + '__lr_mult__': '0.0', + '__wd_mult__': '0.0' + }, + name='ssh_m2_red_upsampling') + #conv5_128_up = mx.symbol.UpSampling(conv5_128, scale=2, sample_type='nearest', workspace=512, name='ssh_m2_red_up', num_args=1) + conv4_128 = mx.symbol.Crop(*[conv4_128, conv5_128_up]) + #conv5_128_up = mx.symbol.Crop(*[conv5_128_up, conv4_128]) + + conv_sum = conv4_128 + conv5_128_up + #conv_sum = conv_1x1 + + m1_conv = conv_act_layer(conv_sum, + 'ssh_m1_conv', + F2, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + m1 = ssh_detection_module(m1_conv, F2, 'ssh_m1_det') + m2 = ssh_detection_module(relu5_3, F1, 'ssh_m2_det') + m3 = ssh_detection_module(m3_pool, F1, 'ssh_m3_det') + return {8: m1, 16: m2, 32: m3} + else: + F1 = 256 + F2 = 256 + _bwm = 1.0 + conv4_128 = conv_act_layer(relu4_3, + 'ssh_m1_red_conv', + F2, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + conv5_128 = conv_act_layer(relu5_3, + 'ssh_m2_red_conv', + F2, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + conv5_128_up = mx.symbol.Deconvolution(data=conv5_128, + num_filter=F2, + kernel=(4, 4), + stride=(2, 2), + pad=(1, 1), + num_group=F2, + no_bias=True, + attr={ + '__lr_mult__': '0.0', + '__wd_mult__': '0.0' + }, + name='ssh_m2_red_upsampling') + #conv5_128_up = mx.symbol.UpSampling(conv5_128, scale=2, sample_type='nearest', workspace=512, name='ssh_m2_red_up', num_args=1) + conv4_128 = mx.symbol.Crop(*[conv4_128, conv5_128_up]) + #conv5_128_up = mx.symbol.Crop(*[conv5_128_up, conv4_128]) + + conv_sum = conv4_128 + conv5_128_up + m1_conv = conv_act_layer(conv_sum, + 'ssh_m1_conv', + F2, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + act_type='relu', + bias_wd_mult=_bwm) + m1 = cpm(m1_conv, 'ssh_m1_det') + m2 = cpm(relu5_3, 'ssh_m2_det') + m3 = cpm(m3_pool, 'ssh_m3_det') + return {8: m1, 16: m2, 32: m3} + + +def get_out(conv_fpn_feat, prefix, stride, landmark=False, lr_mult=1.0): + A = config.NUM_ANCHORS + ret_group = [] + num_anchors = config.RPN_ANCHOR_CFG[str(stride)]['NUM_ANCHORS'] + label = mx.symbol.Variable(name='%s_label_stride%d' % (prefix, stride)) + bbox_target = mx.symbol.Variable(name='%s_bbox_target_stride%d' % + (prefix, stride)) + bbox_weight = mx.symbol.Variable(name='%s_bbox_weight_stride%d' % + (prefix, stride)) + if landmark: + landmark_target = mx.symbol.Variable( + name='%s_landmark_target_stride%d' % (prefix, stride)) + landmark_weight = mx.symbol.Variable( + name='%s_landmark_weight_stride%d' % (prefix, stride)) + rpn_relu = conv_fpn_feat[stride] + maxout_stat = 0 + if config.USE_MAXOUT >= 1 and stride == config.RPN_FEAT_STRIDE[-1]: + maxout_stat = 1 + if config.USE_MAXOUT >= 2 and stride != config.RPN_FEAT_STRIDE[-1]: + maxout_stat = 2 + + if maxout_stat == 0: + rpn_cls_score = conv_act_layer(rpn_relu, + '%s_rpn_cls_score_stride%d' % + (prefix, stride), + 2 * num_anchors, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='') + elif maxout_stat == 1: + cls_list = [] + for a in range(num_anchors): + rpn_cls_score_bg = conv_act_layer( + rpn_relu, + '%s_rpn_cls_score_stride%d_anchor%d_bg' % (prefix, stride, a), + 3, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='') + rpn_cls_score_bg = mx.sym.max(rpn_cls_score_bg, + axis=1, + keepdims=True) + cls_list.append(rpn_cls_score_bg) + rpn_cls_score_fg = conv_act_layer( + rpn_relu, + '%s_rpn_cls_score_stride%d_anchor%d_fg' % (prefix, stride, a), + 1, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='') + cls_list.append(rpn_cls_score_fg) + rpn_cls_score = mx.sym.concat(*cls_list, + dim=1, + name='%s_rpn_cls_score_stride%d' % + (prefix, stride)) + else: + cls_list = [] + for a in range(num_anchors): + rpn_cls_score_bg = conv_act_layer( + rpn_relu, + '%s_rpn_cls_score_stride%d_anchor%d_bg' % (prefix, stride, a), + 1, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='') + cls_list.append(rpn_cls_score_bg) + rpn_cls_score_fg = conv_act_layer( + rpn_relu, + '%s_rpn_cls_score_stride%d_anchor%d_fg' % (prefix, stride, a), + 3, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='') + rpn_cls_score_fg = mx.sym.max(rpn_cls_score_fg, + axis=1, + keepdims=True) + cls_list.append(rpn_cls_score_fg) + rpn_cls_score = mx.sym.concat(*cls_list, + dim=1, + name='%s_rpn_cls_score_stride%d' % + (prefix, stride)) + + rpn_bbox_pred = conv_act_layer(rpn_relu, + '%s_rpn_bbox_pred_stride%d' % + (prefix, stride), + 4 * num_anchors, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='') + + # prepare rpn data + rpn_cls_score_reshape = mx.symbol.Reshape( + data=rpn_cls_score, + shape=(0, 2, -1), + name="%s_rpn_cls_score_reshape_stride%s" % (prefix, stride)) + rpn_bbox_pred_reshape = mx.symbol.Reshape( + data=rpn_bbox_pred, + shape=(0, 0, -1), + name="%s_rpn_bbox_pred_reshape_stride%s" % (prefix, stride)) + if landmark: + rpn_landmark_pred = conv_act_layer(rpn_relu, + '%s_rpn_landmark_pred_stride%d' % + (prefix, stride), + 10 * num_anchors, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + act_type='') + rpn_landmark_pred_reshape = mx.symbol.Reshape( + data=rpn_landmark_pred, + shape=(0, 0, -1), + name="%s_rpn_landmark_pred_reshape_stride%s" % (prefix, stride)) + + if config.TRAIN.RPN_ENABLE_OHEM >= 2: + label, anchor_weight = mx.sym.Custom(op_type='rpn_fpn_ohem3', + stride=int(stride), + network=config.network, + dataset=config.dataset, + prefix=prefix, + cls_score=rpn_cls_score_reshape, + labels=label) + + _bbox_weight = mx.sym.tile(anchor_weight, (1, 1, 4)) + _bbox_weight = _bbox_weight.reshape((0, -1, A * 4)).transpose( + (0, 2, 1)) + bbox_weight = mx.sym.elemwise_mul(bbox_weight, + _bbox_weight, + name='%s_bbox_weight_mul_stride%s' % + (prefix, stride)) + + if landmark: + _landmark_weight = mx.sym.tile(anchor_weight, (1, 1, 10)) + _landmark_weight = _landmark_weight.reshape( + (0, -1, A * 10)).transpose((0, 2, 1)) + landmark_weight = mx.sym.elemwise_mul( + landmark_weight, + _landmark_weight, + name='%s_landmark_weight_mul_stride%s' % (prefix, stride)) + #if not config.FACE_LANDMARK: + # label, bbox_weight = mx.sym.Custom(op_type='rpn_fpn_ohem', stride=int(stride), cls_score=rpn_cls_score_reshape, bbox_weight = bbox_weight , labels = label) + #else: + # label, bbox_weight, landmark_weight = mx.sym.Custom(op_type='rpn_fpn_ohem2', stride=int(stride), cls_score=rpn_cls_score_reshape, bbox_weight = bbox_weight, landmark_weight=landmark_weight, labels = label) + #cls loss + rpn_cls_prob = mx.symbol.SoftmaxOutput(data=rpn_cls_score_reshape, + label=label, + multi_output=True, + normalization='valid', + use_ignore=True, + ignore_label=-1, + grad_scale=lr_mult, + name='%s_rpn_cls_prob_stride%d' % + (prefix, stride)) + ret_group.append(rpn_cls_prob) + ret_group.append(mx.sym.BlockGrad(label)) + + #bbox loss + bbox_diff = rpn_bbox_pred_reshape - bbox_target + bbox_diff = bbox_diff * bbox_weight + rpn_bbox_loss_ = mx.symbol.smooth_l1(name='%s_rpn_bbox_loss_stride%d_' % + (prefix, stride), + scalar=3.0, + data=bbox_diff) + rpn_bbox_loss = mx.sym.MakeLoss( + name='%s_rpn_bbox_loss_stride%d' % (prefix, stride), + data=rpn_bbox_loss_, + grad_scale=1.0 * lr_mult / (config.TRAIN.RPN_BATCH_SIZE)) + ret_group.append(rpn_bbox_loss) + ret_group.append(mx.sym.BlockGrad(bbox_weight)) + + #landmark loss + if landmark: + landmark_diff = rpn_landmark_pred_reshape - landmark_target + landmark_diff = landmark_diff * landmark_weight + rpn_landmark_loss_ = mx.symbol.smooth_l1( + name='%s_rpn_landmark_loss_stride%d_' % (prefix, stride), + scalar=3.0, + data=landmark_diff) + rpn_landmark_loss = mx.sym.MakeLoss( + name='%s_rpn_landmark_loss_stride%d' % (prefix, stride), + data=rpn_landmark_loss_, + grad_scale=0.5 * lr_mult / (config.TRAIN.RPN_BATCH_SIZE)) + ret_group.append(rpn_landmark_loss) + ret_group.append(mx.sym.BlockGrad(landmark_weight)) + return ret_group + + +def get_ssh_train(): + """ + Region Proposal Network with VGG + :return: Symbol + """ + data = mx.symbol.Variable(name="data") + + # shared convolutional layers + conv_fpn_feat = get_ssh_conv(data) + ret_group = [] + for stride in config.RPN_FEAT_STRIDE: + ret = get_out(conv_fpn_feat, + 'face', + stride, + config.FACE_LANDMARK, + lr_mult=1.0) + ret_group += ret + if config.HEAD_BOX: + ret = get_out(conv_fpn_feat, 'head', stride, False, lr_mult=1.0) + ret_group += ret + + return mx.sym.Group(ret_group) diff --git a/RetinaFace/rcnn/tools/__init__.py b/detection/RetinaFace/rcnn/tools/__init__.py similarity index 100% rename from RetinaFace/rcnn/tools/__init__.py rename to detection/RetinaFace/rcnn/tools/__init__.py diff --git a/detection/RetinaFace/rcnn/tools/reeval.py b/detection/RetinaFace/rcnn/tools/reeval.py new file mode 100644 index 0000000..0903a06 --- /dev/null +++ b/detection/RetinaFace/rcnn/tools/reeval.py @@ -0,0 +1,68 @@ +import argparse +try: + import cPickle as pickle +except ImportError: + import pickle +import os +import mxnet as mx + +from ..logger import logger +from ..config import config, default, generate_config +from ..dataset import * + + +def reeval(args): + # load imdb + imdb = eval(args.dataset)(args.image_set, args.root_path, + args.dataset_path) + + # load detection results + cache_file = os.path.join(imdb.cache_path, imdb.name, 'detections.pkl') + with open(cache_file) as f: + detections = pickle.load(f) + + # eval + imdb.evaluate_detections(detections) + + +def parse_args(): + parser = argparse.ArgumentParser(description='imdb test') + # general + parser.add_argument('--network', + help='network name', + default=default.network, + type=str) + parser.add_argument('--dataset', + help='dataset name', + default=default.dataset, + type=str) + args, rest = parser.parse_known_args() + generate_config(args.network, args.dataset) + parser.add_argument('--image_set', + help='image_set name', + default=default.image_set, + type=str) + parser.add_argument('--root_path', + help='output data folder', + default=default.root_path, + type=str) + parser.add_argument('--dataset_path', + help='dataset path', + default=default.dataset_path, + type=str) + # other + parser.add_argument('--no_shuffle', + help='disable random shuffle', + action='store_true') + args = parser.parse_args() + return args + + +def main(): + args = parse_args() + logger.info('Called with argument: %s' % args) + reeval(args) + + +if __name__ == '__main__': + main() diff --git a/detection/RetinaFace/rcnn/tools/test_rcnn.py b/detection/RetinaFace/rcnn/tools/test_rcnn.py new file mode 100644 index 0000000..d571ed6 --- /dev/null +++ b/detection/RetinaFace/rcnn/tools/test_rcnn.py @@ -0,0 +1,161 @@ +import argparse +import pprint +import mxnet as mx + +from ..logger import logger +from ..config import config, default, generate_config +from ..symbol import * +from ..dataset import * +from ..core.loader import TestLoader +from ..core.tester import Predictor, pred_eval +from ..utils.load_model import load_param + + +def test_rcnn(network, dataset, image_set, root_path, dataset_path, ctx, + prefix, epoch, vis, shuffle, has_rpn, proposal, thresh): + # set config + if has_rpn: + config.TEST.HAS_RPN = True + + # print config + logger.info(pprint.pformat(config)) + + # load symbol and testing data + if has_rpn: + sym = eval('get_' + network + '_test')(num_classes=config.NUM_CLASSES, + num_anchors=config.NUM_ANCHORS) + imdb = eval(dataset)(image_set, root_path, dataset_path) + roidb = imdb.gt_roidb() + else: + sym = eval('get_' + network + + '_rcnn_test')(num_classes=config.NUM_CLASSES) + imdb = eval(dataset)(image_set, root_path, dataset_path) + gt_roidb = imdb.gt_roidb() + roidb = eval('imdb.' + proposal + '_roidb')(gt_roidb) + + # get test data iter + test_data = TestLoader(roidb, + batch_size=1, + shuffle=shuffle, + has_rpn=has_rpn) + + # load model + arg_params, aux_params = load_param(prefix, + epoch, + convert=True, + ctx=ctx, + process=True) + + # infer shape + data_shape_dict = dict(test_data.provide_data) + arg_shape, _, aux_shape = sym.infer_shape(**data_shape_dict) + arg_shape_dict = dict(zip(sym.list_arguments(), arg_shape)) + aux_shape_dict = dict(zip(sym.list_auxiliary_states(), aux_shape)) + + # check parameters + for k in sym.list_arguments(): + if k in data_shape_dict or 'label' in k: + continue + assert k in arg_params, k + ' not initialized' + assert arg_params[k].shape == arg_shape_dict[k], \ + 'shape inconsistent for ' + k + ' inferred ' + str(arg_shape_dict[k]) + ' provided ' + str(arg_params[k].shape) + for k in sym.list_auxiliary_states(): + assert k in aux_params, k + ' not initialized' + assert aux_params[k].shape == aux_shape_dict[k], \ + 'shape inconsistent for ' + k + ' inferred ' + str(aux_shape_dict[k]) + ' provided ' + str(aux_params[k].shape) + + # decide maximum shape + data_names = [k[0] for k in test_data.provide_data] + label_names = None + max_data_shape = [('data', (1, 3, max([v[0] for v in config.SCALES]), + max([v[1] for v in config.SCALES])))] + if not has_rpn: + max_data_shape.append( + ('rois', (1, config.TEST.PROPOSAL_POST_NMS_TOP_N + 30, 5))) + + # create predictor + predictor = Predictor(sym, + data_names, + label_names, + context=ctx, + max_data_shapes=max_data_shape, + provide_data=test_data.provide_data, + provide_label=test_data.provide_label, + arg_params=arg_params, + aux_params=aux_params) + + # start detection + pred_eval(predictor, test_data, imdb, vis=vis, thresh=thresh) + + +def parse_args(): + parser = argparse.ArgumentParser(description='Test a Fast R-CNN network') + # general + parser.add_argument('--network', + help='network name', + default=default.network, + type=str) + parser.add_argument('--dataset', + help='dataset name', + default=default.dataset, + type=str) + args, rest = parser.parse_known_args() + generate_config(args.network, args.dataset) + parser.add_argument('--image_set', + help='image_set name', + default=default.test_image_set, + type=str) + parser.add_argument('--root_path', + help='output data folder', + default=default.root_path, + type=str) + parser.add_argument('--dataset_path', + help='dataset path', + default=default.dataset_path, + type=str) + # testing + parser.add_argument('--prefix', + help='model to test with', + default=default.rcnn_prefix, + type=str) + parser.add_argument('--epoch', + help='model to test with', + default=default.rcnn_epoch, + type=int) + parser.add_argument('--gpu', + help='GPU device to test with', + default=0, + type=int) + # rcnn + parser.add_argument('--vis', + help='turn on visualization', + action='store_true') + parser.add_argument('--thresh', + help='valid detection threshold', + default=1e-3, + type=float) + parser.add_argument('--shuffle', + help='shuffle data on visualization', + action='store_true') + parser.add_argument('--has_rpn', + help='generate proposals on the fly', + action='store_true') + parser.add_argument('--proposal', + help='can be ss for selective search or rpn', + default='rpn', + type=str) + args = parser.parse_args() + return args + + +def main(): + args = parse_args() + logger.info('Called with argument: %s' % args) + ctx = mx.gpu(args.gpu) + test_rcnn(args.network, args.dataset, args.image_set, args.root_path, + args.dataset_path, ctx, args.prefix, args.epoch, args.vis, + args.shuffle, args.has_rpn, args.proposal, args.thresh) + + +if __name__ == '__main__': + main() diff --git a/detection/RetinaFace/rcnn/tools/test_rpn.py b/detection/RetinaFace/rcnn/tools/test_rpn.py new file mode 100644 index 0000000..e770406 --- /dev/null +++ b/detection/RetinaFace/rcnn/tools/test_rpn.py @@ -0,0 +1,160 @@ +import argparse +import pprint +import mxnet as mx + +from ..logger import logger +from ..config import config, default, generate_config +from ..symbol import * +from ..dataset import * +from ..core.loader import TestLoader +from ..core.tester import Predictor, generate_proposals, test_proposals +from ..utils.load_model import load_param + + +def test_rpn(network, + dataset, + image_set, + root_path, + dataset_path, + ctx, + prefix, + epoch, + vis, + shuffle, + thresh, + test_output=False): + # rpn generate proposal config + config.TEST.HAS_RPN = True + + # print config + logger.info(pprint.pformat(config)) + + # load symbol + sym = eval('get_' + network + '_rpn_test')() + + # load dataset and prepare imdb for training + imdb = eval(dataset)(image_set, root_path, dataset_path) + roidb = imdb.gt_roidb() + test_data = TestLoader(roidb, + batch_size=1, + shuffle=shuffle, + has_rpn=True, + withlabel=True) + + # load model + arg_params, aux_params = load_param(prefix, epoch, convert=True, ctx=ctx) + + # infer shape + data_shape_dict = dict(test_data.provide_data) + arg_shape, _, aux_shape = sym.infer_shape(**data_shape_dict) + arg_shape_dict = dict(zip(sym.list_arguments(), arg_shape)) + aux_shape_dict = dict(zip(sym.list_auxiliary_states(), aux_shape)) + + # check parameters + for k in sym.list_arguments(): + if k in data_shape_dict or 'label' in k: + continue + assert k in arg_params, k + ' not initialized' + assert arg_params[k].shape == arg_shape_dict[k], \ + 'shape inconsistent for ' + k + ' inferred ' + str(arg_shape_dict[k]) + ' provided ' + str(arg_params[k].shape) + for k in sym.list_auxiliary_states(): + assert k in aux_params, k + ' not initialized' + assert aux_params[k].shape == aux_shape_dict[k], \ + 'shape inconsistent for ' + k + ' inferred ' + str(aux_shape_dict[k]) + ' provided ' + str(aux_params[k].shape) + + # decide maximum shape + data_names = [k[0] for k in test_data.provide_data] + label_names = None if test_data.provide_label is None else [ + k[0] for k in test_data.provide_label + ] + max_data_shape = [('data', (1, 3, max([v[1] for v in config.SCALES]), + max([v[1] for v in config.SCALES])))] + + # create predictor + predictor = Predictor(sym, + data_names, + label_names, + context=ctx, + max_data_shapes=max_data_shape, + provide_data=test_data.provide_data, + provide_label=test_data.provide_label, + arg_params=arg_params, + aux_params=aux_params) + + # start testing + if not test_output: + imdb_boxes = generate_proposals(predictor, + test_data, + imdb, + vis=vis, + thresh=thresh) + imdb.evaluate_recall(roidb, candidate_boxes=imdb_boxes) + else: + test_proposals(predictor, test_data, imdb, roidb, vis=vis) + + +def parse_args(): + parser = argparse.ArgumentParser( + description='Test a Region Proposal Network') + # general + parser.add_argument('--network', + help='network name', + default=default.network, + type=str) + parser.add_argument('--dataset', + help='dataset name', + default=default.dataset, + type=str) + args, rest = parser.parse_known_args() + generate_config(args.network, args.dataset) + parser.add_argument('--image_set', + help='image_set name', + default=default.test_image_set, + type=str) + parser.add_argument('--root_path', + help='output data folder', + default=default.root_path, + type=str) + parser.add_argument('--dataset_path', + help='dataset path', + default=default.dataset_path, + type=str) + # testing + parser.add_argument('--prefix', + help='model to test with', + default=default.rpn_prefix, + type=str) + parser.add_argument('--epoch', + help='model to test with', + default=default.rpn_epoch, + type=int) + # rpn + parser.add_argument('--gpu', + help='GPU device to test with', + default=0, + type=int) + parser.add_argument('--vis', + help='turn on visualization', + action='store_true') + parser.add_argument('--thresh', + help='rpn proposal threshold', + default=0, + type=float) + parser.add_argument('--shuffle', + help='shuffle data on visualization', + action='store_true') + args = parser.parse_args() + return args + + +def main(): + args = parse_args() + logger.info('Called with argument: %s' % args) + ctx = mx.gpu(args.gpu) + test_rpn(args.network, args.dataset, args.image_set, args.root_path, + args.dataset_path, ctx, args.prefix, args.epoch, args.vis, + args.shuffle, args.thresh) + + +if __name__ == '__main__': + main() diff --git a/detection/RetinaFace/rcnn/tools/train_rcnn.py b/detection/RetinaFace/rcnn/tools/train_rcnn.py new file mode 100644 index 0000000..167a3b0 --- /dev/null +++ b/detection/RetinaFace/rcnn/tools/train_rcnn.py @@ -0,0 +1,286 @@ +import argparse +import pprint +import mxnet as mx + +from ..logger import logger +from ..config import config, default, generate_config +from ..symbol import * +from ..core import callback, metric +from ..core.loader import ROIIter +from ..core.module import MutableModule +from ..processing.bbox_regression import add_bbox_regression_targets +from ..utils.load_data import load_proposal_roidb, merge_roidb, filter_roidb +from ..utils.load_model import load_param + + +def train_rcnn(network, dataset, image_set, root_path, dataset_path, frequent, + kvstore, work_load_list, no_flip, no_shuffle, resume, ctx, + pretrained, epoch, prefix, begin_epoch, end_epoch, train_shared, + lr, lr_step, proposal): + # set up config + config.TRAIN.BATCH_IMAGES = 2 + config.TRAIN.BATCH_ROIS = 128 + if proposal == 'ss': + config.TRAIN.BG_THRESH_LO = 0.1 # reproduce Fast R-CNN + + # load symbol + sym = eval('get_' + network + '_rcnn')(num_classes=config.NUM_CLASSES) + + # setup multi-gpu + batch_size = len(ctx) + input_batch_size = config.TRAIN.BATCH_IMAGES * batch_size + + # print config + logger.info(pprint.pformat(config)) + + # load dataset and prepare imdb for training + image_sets = [iset for iset in image_set.split('+')] + roidbs = [ + load_proposal_roidb(dataset, + image_set, + root_path, + dataset_path, + proposal=proposal, + append_gt=True, + flip=not no_flip) for image_set in image_sets + ] + roidb = merge_roidb(roidbs) + roidb = filter_roidb(roidb) + means, stds = add_bbox_regression_targets(roidb) + + # load training data + train_data = ROIIter(roidb, + batch_size=input_batch_size, + shuffle=not no_shuffle, + ctx=ctx, + work_load_list=work_load_list, + aspect_grouping=config.TRAIN.ASPECT_GROUPING) + + # infer max shape + max_data_shape = [('data', (input_batch_size, 3, + max([v[0] for v in config.SCALES]), + max([v[1] for v in config.SCALES])))] + logger.info('providing maximum shape %s' % max_data_shape) + + # infer shape + data_shape_dict = dict(train_data.provide_data + train_data.provide_label) + arg_shape, out_shape, aux_shape = sym.infer_shape(**data_shape_dict) + arg_shape_dict = dict(zip(sym.list_arguments(), arg_shape)) + out_shape_dict = dict(zip(sym.list_outputs(), out_shape)) + aux_shape_dict = dict(zip(sym.list_auxiliary_states(), aux_shape)) + logger.info('output shape %s' % pprint.pformat(out_shape_dict)) + + # load and initialize params + if resume: + arg_params, aux_params = load_param(prefix, begin_epoch, convert=True) + else: + arg_params, aux_params = load_param(pretrained, epoch, convert=True) + arg_params['cls_score_weight'] = mx.random.normal( + 0, 0.01, shape=arg_shape_dict['cls_score_weight']) + arg_params['cls_score_bias'] = mx.nd.zeros( + shape=arg_shape_dict['cls_score_bias']) + arg_params['bbox_pred_weight'] = mx.random.normal( + 0, 0.001, shape=arg_shape_dict['bbox_pred_weight']) + arg_params['bbox_pred_bias'] = mx.nd.zeros( + shape=arg_shape_dict['bbox_pred_bias']) + + # check parameter shapes + for k in sym.list_arguments(): + if k in data_shape_dict: + continue + assert k in arg_params, k + ' not initialized' + assert arg_params[k].shape == arg_shape_dict[k], \ + 'shape inconsistent for ' + k + ' inferred ' + str(arg_shape_dict[k]) + ' provided ' + str(arg_params[k].shape) + for k in sym.list_auxiliary_states(): + assert k in aux_params, k + ' not initialized' + assert aux_params[k].shape == aux_shape_dict[k], \ + 'shape inconsistent for ' + k + ' inferred ' + str(aux_shape_dict[k]) + ' provided ' + str(aux_params[k].shape) + + # prepare training + # create solver + data_names = [k[0] for k in train_data.provide_data] + label_names = [k[0] for k in train_data.provide_label] + if train_shared: + fixed_param_prefix = config.FIXED_PARAMS_SHARED + else: + fixed_param_prefix = config.FIXED_PARAMS + mod = MutableModule(sym, + data_names=data_names, + label_names=label_names, + logger=logger, + context=ctx, + work_load_list=work_load_list, + max_data_shapes=max_data_shape, + fixed_param_prefix=fixed_param_prefix) + + # decide training params + # metric + eval_metric = metric.RCNNAccMetric() + cls_metric = metric.RCNNLogLossMetric() + bbox_metric = metric.RCNNL1LossMetric() + eval_metrics = mx.metric.CompositeEvalMetric() + for child_metric in [eval_metric, cls_metric, bbox_metric]: + eval_metrics.add(child_metric) + # callback + batch_end_callback = mx.callback.Speedometer(train_data.batch_size, + frequent=frequent, + auto_reset=False) + epoch_end_callback = callback.do_checkpoint(prefix, means, stds) + # decide learning rate + base_lr = lr + lr_factor = 0.1 + lr_epoch = [int(epoch) for epoch in lr_step.split(',')] + lr_epoch_diff = [ + epoch - begin_epoch for epoch in lr_epoch if epoch > begin_epoch + ] + lr = base_lr * (lr_factor**(len(lr_epoch) - len(lr_epoch_diff))) + lr_iters = [ + int(epoch * len(roidb) / batch_size) for epoch in lr_epoch_diff + ] + logger.info('lr %f lr_epoch_diff %s lr_iters %s' % + (lr, lr_epoch_diff, lr_iters)) + lr_scheduler = mx.lr_scheduler.MultiFactorScheduler(lr_iters, lr_factor) + # optimizer + optimizer_params = { + 'momentum': 0.9, + 'wd': 0.0005, + 'learning_rate': lr, + 'lr_scheduler': lr_scheduler, + 'rescale_grad': (1.0 / batch_size), + 'clip_gradient': 5 + } + + # train + mod.fit(train_data, + eval_metric=eval_metrics, + epoch_end_callback=epoch_end_callback, + batch_end_callback=batch_end_callback, + kvstore=kvstore, + optimizer='sgd', + optimizer_params=optimizer_params, + arg_params=arg_params, + aux_params=aux_params, + begin_epoch=begin_epoch, + num_epoch=end_epoch) + + +def parse_args(): + parser = argparse.ArgumentParser(description='Train a Fast R-CNN Network') + # general + parser.add_argument('--network', + help='network name', + default=default.network, + type=str) + parser.add_argument('--dataset', + help='dataset name', + default=default.dataset, + type=str) + args, rest = parser.parse_known_args() + generate_config(args.network, args.dataset) + parser.add_argument('--image_set', + help='image_set name', + default=default.image_set, + type=str) + parser.add_argument('--root_path', + help='output data folder', + default=default.root_path, + type=str) + parser.add_argument('--dataset_path', + help='dataset path', + default=default.dataset_path, + type=str) + # training + parser.add_argument('--frequent', + help='frequency of logging', + default=default.frequent, + type=int) + parser.add_argument('--kvstore', + help='the kv-store type', + default=default.kvstore, + type=str) + parser.add_argument('--work_load_list', + help='work load for different devices', + default=None, + type=list) + parser.add_argument('--no_flip', + help='disable flip images', + action='store_true') + parser.add_argument('--no_shuffle', + help='disable random shuffle', + action='store_true') + parser.add_argument('--resume', + help='continue training', + action='store_true') + # rcnn + parser.add_argument('--gpus', + help='GPU device to train with', + default='0', + type=str) + parser.add_argument('--pretrained', + help='pretrained model prefix', + default=default.pretrained, + type=str) + parser.add_argument('--pretrained_epoch', + help='pretrained model epoch', + default=default.pretrained_epoch, + type=int) + parser.add_argument('--prefix', + help='new model prefix', + default=default.rcnn_prefix, + type=str) + parser.add_argument('--begin_epoch', + help='begin epoch of training', + default=0, + type=int) + parser.add_argument('--end_epoch', + help='end epoch of training', + default=default.rcnn_epoch, + type=int) + parser.add_argument('--lr', + help='base learning rate', + default=default.rcnn_lr, + type=float) + parser.add_argument('--lr_step', + help='learning rate steps (in epoch)', + default=default.rcnn_lr_step, + type=str) + parser.add_argument('--train_shared', + help='second round train shared params', + action='store_true') + parser.add_argument('--proposal', + help='can be ss for selective search or rpn', + default='rpn', + type=str) + args = parser.parse_args() + return args + + +def main(): + args = parse_args() + logger.info('Called with argument: %s' % args) + ctx = [mx.gpu(int(i)) for i in args.gpus.split(',')] + train_rcnn(args.network, + args.dataset, + args.image_set, + args.root_path, + args.dataset_path, + args.frequent, + args.kvstore, + args.work_load_list, + args.no_flip, + args.no_shuffle, + args.resume, + ctx, + args.pretrained, + args.pretrained_epoch, + args.prefix, + args.begin_epoch, + args.end_epoch, + train_shared=args.train_shared, + lr=args.lr, + lr_step=args.lr_step, + proposal=args.proposal) + + +if __name__ == '__main__': + main() diff --git a/detection/RetinaFace/rcnn/tools/train_rpn.py b/detection/RetinaFace/rcnn/tools/train_rpn.py new file mode 100644 index 0000000..9d60080 --- /dev/null +++ b/detection/RetinaFace/rcnn/tools/train_rpn.py @@ -0,0 +1,300 @@ +import argparse +import logging +import pprint +import mxnet as mx + +from ..config import config, default, generate_config +from ..symbol import * +from ..core import callback, metric +from ..core.loader import AnchorLoaderFPN +from ..core.module import MutableModule +from ..utils.load_data import load_gt_roidb, merge_roidb, filter_roidb +from ..utils.load_model import load_param + + +def train_rpn(network, dataset, image_set, root_path, dataset_path, frequent, + kvstore, work_load_list, no_flip, no_shuffle, resume, ctx, + pretrained, epoch, prefix, begin_epoch, end_epoch, train_shared, + lr, lr_step): + # set up logger + logging.basicConfig() + logger = logging.getLogger() + logger.setLevel(logging.INFO) + + # setup config + assert config.TRAIN.BATCH_IMAGES == 1 + + # load symbol + sym = eval('get_' + network + '_rpn')() + feat_sym = [] + for stride in config.RPN_FEAT_STRIDE: + feat_sym.append(sym.get_internals()['rpn_cls_score_stride%s_output' % + stride]) + + # setup multi-gpu + batch_size = len(ctx) + input_batch_size = config.TRAIN.BATCH_IMAGES * batch_size + + # print config + pprint.pprint(config) + + # load dataset and prepare imdb for training + image_sets = [iset for iset in image_set.split('+')] + roidbs = [ + load_gt_roidb(dataset, + image_set, + root_path, + dataset_path, + flip=not no_flip) for image_set in image_sets + ] + roidb = merge_roidb(roidbs) + roidb = filter_roidb(roidb) + + # load training data + #train_data = AnchorLoaderFPN(feat_sym, roidb, batch_size=input_batch_size, shuffle=not no_shuffle, + # ctx=ctx, work_load_list=work_load_list, + # feat_stride=config.RPN_FEAT_STRIDE, anchor_scales=config.ANCHOR_SCALES, + # anchor_ratios=config.ANCHOR_RATIOS, aspect_grouping=config.TRAIN.ASPECT_GROUPING, + # allowed_border=9999) + train_data = AnchorLoaderFPN(feat_sym, + roidb, + batch_size=input_batch_size, + shuffle=not no_shuffle, + ctx=ctx, + work_load_list=work_load_list) + + # infer max shape + max_data_shape = [('data', (input_batch_size, 3, + max([v[0] for v in config.SCALES]), + max([v[1] for v in config.SCALES])))] + max_data_shape, max_label_shape = train_data.infer_shape(max_data_shape) + print 'providing maximum shape', max_data_shape, max_label_shape + + # infer shape + data_shape_dict = dict(train_data.provide_data + train_data.provide_label) + arg_shape, out_shape, aux_shape = sym.infer_shape(**data_shape_dict) + arg_shape_dict = dict(zip(sym.list_arguments(), arg_shape)) + out_shape_dict = zip(sym.list_outputs(), out_shape) + aux_shape_dict = dict(zip(sym.list_auxiliary_states(), aux_shape)) + print 'output shape' + pprint.pprint(out_shape_dict) + + # load and initialize params + if resume: + arg_params, aux_params = load_param(prefix, begin_epoch, convert=True) + else: + arg_params, aux_params = load_param(pretrained, epoch, convert=True) + init = mx.init.Xavier(factor_type="in", + rnd_type='gaussian', + magnitude=2) + init_internal = mx.init.Normal(sigma=0.01) + for k in sym.list_arguments(): + if k in data_shape_dict: + continue + if k not in arg_params: + print 'init', k + arg_params[k] = mx.nd.zeros(shape=arg_shape_dict[k]) + if not k.endswith('bias'): + init_internal(k, arg_params[k]) + + for k in sym.list_auxiliary_states(): + if k not in aux_params: + print 'init', k + aux_params[k] = mx.nd.zeros(shape=aux_shape_dict[k]) + init(k, aux_params[k]) + + # check parameter shapes + for k in sym.list_arguments(): + if k in data_shape_dict: + continue + assert k in arg_params, k + ' not initialized' + assert arg_params[k].shape == arg_shape_dict[k], \ + 'shape inconsistent for ' + k + ' inferred ' + str(arg_shape_dict[k]) + ' provided ' + str(arg_params[k].shape) + for k in sym.list_auxiliary_states(): + assert k in aux_params, k + ' not initialized' + assert aux_params[k].shape == aux_shape_dict[k], \ + 'shape inconsistent for ' + k + ' inferred ' + str(aux_shape_dict[k]) + ' provided ' + str(aux_params[k].shape) + + # create solver + data_names = [k[0] for k in train_data.provide_data] + label_names = [k[0] for k in train_data.provide_label] + if train_shared: + fixed_param_prefix = config.FIXED_PARAMS_SHARED + else: + fixed_param_prefix = config.FIXED_PARAMS + mod = MutableModule(sym, + data_names=data_names, + label_names=label_names, + logger=logger, + context=ctx, + work_load_list=work_load_list, + max_data_shapes=max_data_shape, + max_label_shapes=max_label_shape, + fixed_param_prefix=fixed_param_prefix) + + # decide training params + # metric + eval_metric = metric.RPNAccMetric() + cls_metric = metric.RPNLogLossMetric() + bbox_metric = metric.RPNL1LossMetric() + eval_metrics = mx.metric.CompositeEvalMetric() + for child_metric in [eval_metric, cls_metric, bbox_metric]: + eval_metrics.add(child_metric) + # callback + batch_end_callback = [] + batch_end_callback.append( + mx.callback.Speedometer(train_data.batch_size, frequent=frequent)) + epoch_end_callback = mx.callback.do_checkpoint(prefix) + # decide learning rate + base_lr = lr + lr_factor = 0.1 + lr_epoch = [int(epoch) for epoch in lr_step.split(',')] + lr_epoch_diff = [ + epoch - begin_epoch for epoch in lr_epoch if epoch > begin_epoch + ] + lr = base_lr * (lr_factor**(len(lr_epoch) - len(lr_epoch_diff))) + lr_iters = [ + int(epoch * len(roidb) / batch_size) for epoch in lr_epoch_diff + ] + print 'lr', lr, 'lr_epoch_diff', lr_epoch_diff, 'lr_iters', lr_iters + lr_scheduler = mx.lr_scheduler.MultiFactorScheduler(lr_iters, lr_factor) + # optimizer + optimizer_params = { + 'momentum': 0.9, + 'wd': 0.0001, + 'learning_rate': lr, + 'lr_scheduler': lr_scheduler, + 'rescale_grad': (1.0 / batch_size), + 'clip_gradient': 5 + } + + # train + mod.fit(train_data, + eval_metric=eval_metrics, + epoch_end_callback=epoch_end_callback, + batch_end_callback=batch_end_callback, + kvstore=kvstore, + optimizer='sgd', + optimizer_params=optimizer_params, + arg_params=arg_params, + aux_params=aux_params, + begin_epoch=begin_epoch, + num_epoch=end_epoch) + + +def parse_args(): + parser = argparse.ArgumentParser( + description='Train a Region Proposal Network') + # general + parser.add_argument('--network', + help='network name', + default=default.network, + type=str) + parser.add_argument('--dataset', + help='dataset name', + default=default.dataset, + type=str) + args, rest = parser.parse_known_args() + generate_config(args.network, args.dataset) + parser.add_argument('--image_set', + help='image_set name', + default=default.image_set, + type=str) + parser.add_argument('--root_path', + help='output data folder', + default=default.root_path, + type=str) + parser.add_argument('--dataset_path', + help='dataset path', + default=default.dataset_path, + type=str) + # training + parser.add_argument('--frequent', + help='frequency of logging', + default=default.frequent, + type=int) + parser.add_argument('--kvstore', + help='the kv-store type', + default=default.kvstore, + type=str) + parser.add_argument('--work_load_list', + help='work load for different devices', + default=None, + type=list) + parser.add_argument('--no_flip', + help='disable flip images', + action='store_true') + parser.add_argument('--no_shuffle', + help='disable random shuffle', + action='store_true') + parser.add_argument('--resume', + help='continue training', + action='store_true') + # rpn + parser.add_argument('--gpus', + help='GPU device to train with', + default='0', + type=str) + parser.add_argument('--pretrained', + help='pretrained model prefix', + default=default.pretrained, + type=str) + parser.add_argument('--pretrained_epoch', + help='pretrained model epoch', + default=default.pretrained_epoch, + type=int) + parser.add_argument('--prefix', + help='new model prefix', + default=default.rpn_prefix, + type=str) + parser.add_argument('--begin_epoch', + help='begin epoch of training', + default=0, + type=int) + parser.add_argument('--end_epoch', + help='end epoch of training', + default=default.rpn_epoch, + type=int) + parser.add_argument('--lr', + help='base learning rate', + default=default.rpn_lr, + type=float) + parser.add_argument('--lr_step', + help='learning rate steps (in epoch)', + default=default.rpn_lr_step, + type=str) + parser.add_argument('--train_shared', + help='second round train shared params', + action='store_true') + args = parser.parse_args() + return args + + +def main(): + args = parse_args() + print 'Called with argument:', args + ctx = [mx.gpu(int(i)) for i in args.gpus.split(',')] + train_rpn(args.network, + args.dataset, + args.image_set, + args.root_path, + args.dataset_path, + args.frequent, + args.kvstore, + args.work_load_list, + args.no_flip, + args.no_shuffle, + args.resume, + ctx, + args.pretrained, + args.pretrained_epoch, + args.prefix, + args.begin_epoch, + args.end_epoch, + train_shared=args.train_shared, + lr=args.lr, + lr_step=args.lr_step) + + +if __name__ == '__main__': + main() diff --git a/RetinaFace/rcnn/utils/__init__.py b/detection/RetinaFace/rcnn/utils/__init__.py similarity index 100% rename from RetinaFace/rcnn/utils/__init__.py rename to detection/RetinaFace/rcnn/utils/__init__.py diff --git a/RetinaFace/rcnn/utils/combine_model.py b/detection/RetinaFace/rcnn/utils/combine_model.py similarity index 100% rename from RetinaFace/rcnn/utils/combine_model.py rename to detection/RetinaFace/rcnn/utils/combine_model.py diff --git a/RetinaFace/rcnn/utils/load_data.py b/detection/RetinaFace/rcnn/utils/load_data.py similarity index 74% rename from RetinaFace/rcnn/utils/load_data.py rename to detection/RetinaFace/rcnn/utils/load_data.py index 180814c..5c9d3a4 100644 --- a/RetinaFace/rcnn/utils/load_data.py +++ b/detection/RetinaFace/rcnn/utils/load_data.py @@ -4,7 +4,10 @@ from ..config import config from ..dataset import * -def load_gt_roidb(dataset_name, image_set_name, root_path, dataset_path, +def load_gt_roidb(dataset_name, + image_set_name, + root_path, + dataset_path, flip=False): """ load ground truth roidb """ imdb = eval(dataset_name)(image_set_name, root_path, dataset_path) @@ -16,8 +19,13 @@ def load_gt_roidb(dataset_name, image_set_name, root_path, dataset_path, return roidb -def load_proposal_roidb(dataset_name, image_set_name, root_path, dataset_path, - proposal='rpn', append_gt=True, flip=False): +def load_proposal_roidb(dataset_name, + image_set_name, + root_path, + dataset_path, + proposal='rpn', + append_gt=True, + flip=False): """ load proposal roidb (append_gt when training) """ imdb = eval(dataset_name)(image_set_name, root_path, dataset_path) gt_roidb = imdb.gt_roidb() @@ -37,12 +45,12 @@ def merge_roidb(roidbs): def filter_roidb(roidb): """ remove roidb entries without usable rois """ - def is_valid(entry): """ valid images have at least 1 fg or bg roi """ overlaps = entry['max_overlaps'] fg_inds = np.where(overlaps >= config.TRAIN.FG_THRESH)[0] - bg_inds = np.where((overlaps < config.TRAIN.BG_THRESH_HI) & (overlaps >= config.TRAIN.BG_THRESH_LO))[0] + bg_inds = np.where((overlaps < config.TRAIN.BG_THRESH_HI) + & (overlaps >= config.TRAIN.BG_THRESH_LO))[0] valid = len(fg_inds) > 0 or len(bg_inds) > 0 #valid = len(fg_inds) > 0 return valid @@ -50,6 +58,7 @@ def filter_roidb(roidb): num = len(roidb) filtered_roidb = [entry for entry in roidb if is_valid(entry)] num_after = len(filtered_roidb) - logger.info('load data: filtered %d roidb entries: %d -> %d' % (num - num_after, num, num_after)) + logger.info('load data: filtered %d roidb entries: %d -> %d' % + (num - num_after, num, num_after)) return filtered_roidb diff --git a/RetinaFace/rcnn/utils/load_model.py b/detection/RetinaFace/rcnn/utils/load_model.py similarity index 100% rename from RetinaFace/rcnn/utils/load_model.py rename to detection/RetinaFace/rcnn/utils/load_model.py diff --git a/RetinaFace/rcnn/utils/save_model.py b/detection/RetinaFace/rcnn/utils/save_model.py similarity index 81% rename from RetinaFace/rcnn/utils/save_model.py rename to detection/RetinaFace/rcnn/utils/save_model.py index 1c98869..9c595f4 100644 --- a/RetinaFace/rcnn/utils/save_model.py +++ b/detection/RetinaFace/rcnn/utils/save_model.py @@ -12,7 +12,7 @@ def save_checkpoint(prefix, epoch, arg_params, aux_params): :return: None prefix-epoch.params will be saved for parameters. """ - save_dict = {('arg:%s' % k) : v for k, v in arg_params.items()} - save_dict.update({('aux:%s' % k) : v for k, v in aux_params.items()}) + save_dict = {('arg:%s' % k): v for k, v in arg_params.items()} + save_dict.update({('aux:%s' % k): v for k, v in aux_params.items()}) param_name = '%s-%04d.params' % (prefix, epoch) mx.nd.save(param_name, save_dict) diff --git a/detection/RetinaFace/retinaface.py b/detection/RetinaFace/retinaface.py new file mode 100644 index 0000000..5b274e3 --- /dev/null +++ b/detection/RetinaFace/retinaface.py @@ -0,0 +1,839 @@ +from __future__ import print_function +import sys +import os +import datetime +import time +import numpy as np +import mxnet as mx +from mxnet import ndarray as nd +import cv2 +#from rcnn import config +from rcnn.logger import logger +#from rcnn.processing.bbox_transform import nonlinear_pred, clip_boxes, landmark_pred +from rcnn.processing.bbox_transform import clip_boxes +from rcnn.processing.generate_anchor import generate_anchors_fpn, anchors_plane +from rcnn.processing.nms import gpu_nms_wrapper, cpu_nms_wrapper +from rcnn.processing.bbox_transform import bbox_overlaps + + +class RetinaFace: + def __init__(self, + prefix, + epoch, + ctx_id=0, + network='net3', + nms=0.4, + nocrop=False, + decay4=0.5, + vote=False): + self.ctx_id = ctx_id + self.network = network + self.decay4 = decay4 + self.nms_threshold = nms + self.vote = vote + self.nocrop = nocrop + self.debug = False + self.fpn_keys = [] + self.anchor_cfg = None + pixel_means = [0.0, 0.0, 0.0] + pixel_stds = [1.0, 1.0, 1.0] + pixel_scale = 1.0 + self.preprocess = False + _ratio = (1., ) + fmc = 3 + if network == 'ssh' or network == 'vgg': + pixel_means = [103.939, 116.779, 123.68] + self.preprocess = True + elif network == 'net3': + _ratio = (1., ) + elif network == 'net3a': + _ratio = (1., 1.5) + elif network == 'net6': #like pyramidbox or s3fd + fmc = 6 + elif network == 'net5': #retinaface + fmc = 5 + elif network == 'net5a': + fmc = 5 + _ratio = (1., 1.5) + elif network == 'net4': + fmc = 4 + elif network == 'net4a': + fmc = 4 + _ratio = (1., 1.5) + elif network == 'x5': + fmc = 5 + pixel_means = [103.52, 116.28, 123.675] + pixel_stds = [57.375, 57.12, 58.395] + elif network == 'x3': + fmc = 3 + pixel_means = [103.52, 116.28, 123.675] + pixel_stds = [57.375, 57.12, 58.395] + elif network == 'x3a': + fmc = 3 + _ratio = (1., 1.5) + pixel_means = [103.52, 116.28, 123.675] + pixel_stds = [57.375, 57.12, 58.395] + else: + assert False, 'network setting error %s' % network + + if fmc == 3: + self._feat_stride_fpn = [32, 16, 8] + self.anchor_cfg = { + '32': { + 'SCALES': (32, 16), + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + }, + '16': { + 'SCALES': (8, 4), + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + }, + '8': { + 'SCALES': (2, 1), + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + }, + } + elif fmc == 4: + self._feat_stride_fpn = [32, 16, 8, 4] + self.anchor_cfg = { + '32': { + 'SCALES': (32, 16), + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + }, + '16': { + 'SCALES': (8, 4), + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + }, + '8': { + 'SCALES': (2, 1), + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + }, + '4': { + 'SCALES': (2, 1), + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + }, + } + elif fmc == 6: + self._feat_stride_fpn = [128, 64, 32, 16, 8, 4] + self.anchor_cfg = { + '128': { + 'SCALES': (32, ), + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + }, + '64': { + 'SCALES': (16, ), + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + }, + '32': { + 'SCALES': (8, ), + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + }, + '16': { + 'SCALES': (4, ), + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + }, + '8': { + 'SCALES': (2, ), + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + }, + '4': { + 'SCALES': (1, ), + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + }, + } + elif fmc == 5: + self._feat_stride_fpn = [64, 32, 16, 8, 4] + self.anchor_cfg = {} + _ass = 2.0**(1.0 / 3) + _basescale = 1.0 + for _stride in [4, 8, 16, 32, 64]: + key = str(_stride) + value = { + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + } + scales = [] + for _ in range(3): + scales.append(_basescale) + _basescale *= _ass + value['SCALES'] = tuple(scales) + self.anchor_cfg[key] = value + + print(self._feat_stride_fpn, self.anchor_cfg) + + for s in self._feat_stride_fpn: + self.fpn_keys.append('stride%s' % s) + + dense_anchor = False + #self._anchors_fpn = dict(zip(self.fpn_keys, generate_anchors_fpn(base_size=fpn_base_size, scales=self._scales, ratios=self._ratios))) + self._anchors_fpn = dict( + zip( + self.fpn_keys, + generate_anchors_fpn(dense_anchor=dense_anchor, + cfg=self.anchor_cfg))) + for k in self._anchors_fpn: + v = self._anchors_fpn[k].astype(np.float32) + self._anchors_fpn[k] = v + + self._num_anchors = dict( + zip(self.fpn_keys, + [anchors.shape[0] for anchors in self._anchors_fpn.values()])) + #self._bbox_pred = nonlinear_pred + #self._landmark_pred = landmark_pred + sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) + if self.ctx_id >= 0: + self.ctx = mx.gpu(self.ctx_id) + self.nms = gpu_nms_wrapper(self.nms_threshold, self.ctx_id) + else: + self.ctx = mx.cpu() + self.nms = cpu_nms_wrapper(self.nms_threshold) + self.pixel_means = np.array(pixel_means, dtype=np.float32) + self.pixel_stds = np.array(pixel_stds, dtype=np.float32) + self.pixel_scale = float(pixel_scale) + print('means', self.pixel_means) + self.use_landmarks = False + if len(sym) // len(self._feat_stride_fpn) >= 3: + self.use_landmarks = True + print('use_landmarks', self.use_landmarks) + self.cascade = 0 + if float(len(sym)) // len(self._feat_stride_fpn) > 3.0: + self.cascade = 1 + print('cascade', self.cascade) + #self.bbox_stds = [0.1, 0.1, 0.2, 0.2] + #self.landmark_std = 0.1 + self.bbox_stds = [1.0, 1.0, 1.0, 1.0] + self.landmark_std = 1.0 + + if self.debug: + c = len(sym) // len(self._feat_stride_fpn) + sym = sym[(c * 0):] + self._feat_stride_fpn = [32, 16, 8] + print('sym size:', len(sym)) + + image_size = (640, 640) + self.model = mx.mod.Module(symbol=sym, + context=self.ctx, + label_names=None) + self.model.bind(data_shapes=[('data', (1, 3, image_size[0], + image_size[1]))], + for_training=False) + self.model.set_params(arg_params, aux_params) + + def get_input(self, img): + im = img.astype(np.float32) + im_tensor = np.zeros((1, 3, im.shape[0], im.shape[1])) + for i in range(3): + im_tensor[ + 0, + i, :, :] = (im[:, :, 2 - i] / self.pixel_scale - + self.pixel_means[2 - i]) / self.pixel_stds[2 - i] + #if self.debug: + # timeb = datetime.datetime.now() + # diff = timeb - timea + # print('X2 uses', diff.total_seconds(), 'seconds') + data = nd.array(im_tensor) + return data + + def detect(self, img, threshold=0.5, scales=[1.0], do_flip=False): + #print('in_detect', threshold, scales, do_flip, do_nms) + proposals_list = [] + scores_list = [] + landmarks_list = [] + strides_list = [] + timea = datetime.datetime.now() + flips = [0] + if do_flip: + flips = [0, 1] + + imgs = [img] + if isinstance(img, list): + imgs = img + for img in imgs: + for im_scale in scales: + for flip in flips: + if im_scale != 1.0: + im = cv2.resize(img, + None, + None, + fx=im_scale, + fy=im_scale, + interpolation=cv2.INTER_LINEAR) + else: + im = img.copy() + if flip: + im = im[:, ::-1, :] + if self.nocrop: + if im.shape[0] % 32 == 0: + h = im.shape[0] + else: + h = (im.shape[0] // 32 + 1) * 32 + if im.shape[1] % 32 == 0: + w = im.shape[1] + else: + w = (im.shape[1] // 32 + 1) * 32 + _im = np.zeros((h, w, 3), dtype=np.float32) + _im[0:im.shape[0], 0:im.shape[1], :] = im + im = _im + else: + im = im.astype(np.float32) + if self.debug: + timeb = datetime.datetime.now() + diff = timeb - timea + print('X1 uses', diff.total_seconds(), 'seconds') + #self.model.bind(data_shapes=[('data', (1, 3, image_size[0], image_size[1]))], for_training=False) + #im_info = [im.shape[0], im.shape[1], im_scale] + im_info = [im.shape[0], im.shape[1]] + im_tensor = np.zeros((1, 3, im.shape[0], im.shape[1])) + for i in range(3): + im_tensor[0, i, :, :] = ( + im[:, :, 2 - i] / self.pixel_scale - + self.pixel_means[2 - i]) / self.pixel_stds[2 - i] + if self.debug: + timeb = datetime.datetime.now() + diff = timeb - timea + print('X2 uses', diff.total_seconds(), 'seconds') + data = nd.array(im_tensor) + db = mx.io.DataBatch(data=(data, ), + provide_data=[('data', data.shape)]) + if self.debug: + timeb = datetime.datetime.now() + diff = timeb - timea + print('X3 uses', diff.total_seconds(), 'seconds') + self.model.forward(db, is_train=False) + net_out = self.model.get_outputs() + #post_nms_topN = self._rpn_post_nms_top_n + #min_size_dict = self._rpn_min_size_fpn + + sym_idx = 0 + + for _idx, s in enumerate(self._feat_stride_fpn): + #if len(scales)>1 and s==32 and im_scale==scales[-1]: + # continue + _key = 'stride%s' % s + stride = int(s) + is_cascade = False + if self.cascade: + is_cascade = True + #if self.vote and stride==4 and len(scales)>2 and (im_scale==scales[0]): + # continue + #print('getting', im_scale, stride, idx, len(net_out), data.shape, file=sys.stderr) + scores = net_out[sym_idx].asnumpy() + if self.debug: + timeb = datetime.datetime.now() + diff = timeb - timea + print('A uses', diff.total_seconds(), 'seconds') + #print(scores.shape) + #print('scores',stride, scores.shape, file=sys.stderr) + scores = scores[:, self._num_anchors['stride%s' % + s]:, :, :] + + bbox_deltas = net_out[sym_idx + 1].asnumpy() + + #if DEBUG: + # print 'im_size: ({}, {})'.format(im_info[0], im_info[1]) + # print 'scale: {}'.format(im_info[2]) + + #_height, _width = int(im_info[0] / stride), int(im_info[1] / stride) + height, width = bbox_deltas.shape[ + 2], bbox_deltas.shape[3] + + A = self._num_anchors['stride%s' % s] + K = height * width + anchors_fpn = self._anchors_fpn['stride%s' % s] + anchors = anchors_plane(height, width, stride, + anchors_fpn) + #print((height, width), (_height, _width), anchors.shape, bbox_deltas.shape, scores.shape, file=sys.stderr) + anchors = anchors.reshape((K * A, 4)) + #print('num_anchors', self._num_anchors['stride%s'%s], file=sys.stderr) + #print('HW', (height, width), file=sys.stderr) + #print('anchors_fpn', anchors_fpn.shape, file=sys.stderr) + #print('anchors', anchors.shape, file=sys.stderr) + #print('bbox_deltas', bbox_deltas.shape, file=sys.stderr) + #print('scores', scores.shape, file=sys.stderr) + + #scores = self._clip_pad(scores, (height, width)) + scores = scores.transpose((0, 2, 3, 1)).reshape( + (-1, 1)) + + #print('pre', bbox_deltas.shape, height, width) + #bbox_deltas = self._clip_pad(bbox_deltas, (height, width)) + #print('after', bbox_deltas.shape, height, width) + bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)) + bbox_pred_len = bbox_deltas.shape[3] // A + #print(bbox_deltas.shape) + bbox_deltas = bbox_deltas.reshape((-1, bbox_pred_len)) + bbox_deltas[:, + 0::4] = bbox_deltas[:, 0:: + 4] * self.bbox_stds[0] + bbox_deltas[:, + 1::4] = bbox_deltas[:, 1:: + 4] * self.bbox_stds[1] + bbox_deltas[:, + 2::4] = bbox_deltas[:, 2:: + 4] * self.bbox_stds[2] + bbox_deltas[:, + 3::4] = bbox_deltas[:, 3:: + 4] * self.bbox_stds[3] + proposals = self.bbox_pred(anchors, bbox_deltas) + + #print(anchors.shape, bbox_deltas.shape, A, K, file=sys.stderr) + if is_cascade: + cascade_sym_num = 0 + cls_cascade = False + bbox_cascade = False + __idx = [3, 4] + if not self.use_landmarks: + __idx = [2, 3] + for diff_idx in __idx: + if sym_idx + diff_idx >= len(net_out): + break + body = net_out[sym_idx + diff_idx].asnumpy() + if body.shape[1] // A == 2: #cls branch + if cls_cascade or bbox_cascade: + break + else: + cascade_scores = body[:, self. + _num_anchors[ + 'stride%s' % + s]:, :, :] + cascade_scores = cascade_scores.transpose( + (0, 2, 3, 1)).reshape((-1, 1)) + #scores = (scores+cascade_scores)/2.0 + scores = cascade_scores #TODO? + cascade_sym_num += 1 + cls_cascade = True + #print('find cascade cls at stride', stride) + elif body.shape[1] // A == 4: #bbox branch + cascade_deltas = body.transpose( + (0, 2, 3, 1)).reshape( + (-1, bbox_pred_len)) + cascade_deltas[:, 0:: + 4] = cascade_deltas[:, 0:: + 4] * self.bbox_stds[ + 0] + cascade_deltas[:, 1:: + 4] = cascade_deltas[:, 1:: + 4] * self.bbox_stds[ + 1] + cascade_deltas[:, 2:: + 4] = cascade_deltas[:, 2:: + 4] * self.bbox_stds[ + 2] + cascade_deltas[:, 3:: + 4] = cascade_deltas[:, 3:: + 4] * self.bbox_stds[ + 3] + proposals = self.bbox_pred( + proposals, cascade_deltas) + cascade_sym_num += 1 + bbox_cascade = True + #print('find cascade bbox at stride', stride) + + proposals = clip_boxes(proposals, im_info[:2]) + + #if self.vote: + # if im_scale>1.0: + # keep = self._filter_boxes2(proposals, 160*im_scale, -1) + # else: + # keep = self._filter_boxes2(proposals, -1, 100*im_scale) + # if stride==4: + # keep = self._filter_boxes2(proposals, 12*im_scale, -1) + # proposals = proposals[keep, :] + # scores = scores[keep] + + #keep = self._filter_boxes(proposals, min_size_dict['stride%s'%s] * im_info[2]) + #proposals = proposals[keep, :] + #scores = scores[keep] + #print('333', proposals.shape) + if stride == 4 and self.decay4 < 1.0: + scores *= self.decay4 + + scores_ravel = scores.ravel() + #print('__shapes', proposals.shape, scores_ravel.shape) + #print('max score', np.max(scores_ravel)) + order = np.where(scores_ravel >= threshold)[0] + #_scores = scores_ravel[order] + #_order = _scores.argsort()[::-1] + #order = order[_order] + proposals = proposals[order, :] + scores = scores[order] + if flip: + oldx1 = proposals[:, 0].copy() + oldx2 = proposals[:, 2].copy() + proposals[:, 0] = im.shape[1] - oldx2 - 1 + proposals[:, 2] = im.shape[1] - oldx1 - 1 + + proposals[:, 0:4] /= im_scale + + proposals_list.append(proposals) + scores_list.append(scores) + if self.nms_threshold < 0.0: + _strides = np.empty(shape=(scores.shape), + dtype=np.float32) + _strides.fill(stride) + strides_list.append(_strides) + + if not self.vote and self.use_landmarks: + landmark_deltas = net_out[sym_idx + 2].asnumpy() + #landmark_deltas = self._clip_pad(landmark_deltas, (height, width)) + landmark_pred_len = landmark_deltas.shape[1] // A + landmark_deltas = landmark_deltas.transpose( + (0, 2, 3, 1)).reshape( + (-1, 5, landmark_pred_len // 5)) + landmark_deltas *= self.landmark_std + #print(landmark_deltas.shape, landmark_deltas) + landmarks = self.landmark_pred( + anchors, landmark_deltas) + landmarks = landmarks[order, :] + + if flip: + landmarks[:, :, + 0] = im.shape[1] - landmarks[:, :, + 0] - 1 + #for a in range(5): + # oldx1 = landmarks[:, a].copy() + # landmarks[:,a] = im.shape[1] - oldx1 - 1 + order = [1, 0, 2, 4, 3] + flandmarks = landmarks.copy() + for idx, a in enumerate(order): + flandmarks[:, idx, :] = landmarks[:, a, :] + #flandmarks[:, idx*2] = landmarks[:,a*2] + #flandmarks[:, idx*2+1] = landmarks[:,a*2+1] + landmarks = flandmarks + landmarks[:, :, 0:2] /= im_scale + #landmarks /= im_scale + #landmarks = landmarks.reshape( (-1, landmark_pred_len) ) + landmarks_list.append(landmarks) + #proposals = np.hstack((proposals, landmarks)) + if self.use_landmarks: + sym_idx += 3 + else: + sym_idx += 2 + if is_cascade: + sym_idx += cascade_sym_num + + if self.debug: + timeb = datetime.datetime.now() + diff = timeb - timea + print('B uses', diff.total_seconds(), 'seconds') + proposals = np.vstack(proposals_list) + landmarks = None + if proposals.shape[0] == 0: + if self.use_landmarks: + landmarks = np.zeros((0, 5, 2)) + if self.nms_threshold < 0.0: + return np.zeros((0, 6)), landmarks + else: + return np.zeros((0, 5)), landmarks + scores = np.vstack(scores_list) + #print('shapes', proposals.shape, scores.shape) + scores_ravel = scores.ravel() + order = scores_ravel.argsort()[::-1] + #if config.TEST.SCORE_THRESH>0.0: + # _count = np.sum(scores_ravel>config.TEST.SCORE_THRESH) + # order = order[:_count] + proposals = proposals[order, :] + scores = scores[order] + if self.nms_threshold < 0.0: + strides = np.vstack(strides_list) + strides = strides[order] + if not self.vote and self.use_landmarks: + landmarks = np.vstack(landmarks_list) + landmarks = landmarks[order].astype(np.float32, copy=False) + + if self.nms_threshold > 0.0: + pre_det = np.hstack((proposals[:, 0:4], scores)).astype(np.float32, + copy=False) + if not self.vote: + keep = self.nms(pre_det) + det = np.hstack((pre_det, proposals[:, 4:])) + det = det[keep, :] + if self.use_landmarks: + landmarks = landmarks[keep] + else: + det = np.hstack((pre_det, proposals[:, 4:])) + det = self.bbox_vote(det) + elif self.nms_threshold < 0.0: + det = np.hstack( + (proposals[:, 0:4], scores, strides)).astype(np.float32, + copy=False) + else: + det = np.hstack((proposals[:, 0:4], scores)).astype(np.float32, + copy=False) + + if self.debug: + timeb = datetime.datetime.now() + diff = timeb - timea + print('C uses', diff.total_seconds(), 'seconds') + return det, landmarks + + def detect_center(self, img, threshold=0.5, scales=[1.0], do_flip=False): + det, landmarks = self.detect(img, threshold, scales, do_flip) + if det.shape[0] == 0: + return None, None + bindex = 0 + if det.shape[0] > 1: + img_size = np.asarray(img.shape)[0:2] + bounding_box_size = (det[:, 2] - det[:, 0]) * (det[:, 3] - + det[:, 1]) + img_center = img_size / 2 + offsets = np.vstack([(det[:, 0] + det[:, 2]) / 2 - img_center[1], + (det[:, 1] + det[:, 3]) / 2 - img_center[0]]) + offset_dist_squared = np.sum(np.power(offsets, 2.0), 0) + bindex = np.argmax(bounding_box_size - offset_dist_squared * + 2.0) # some extra weight on the centering + bbox = det[bindex, :] + landmark = landmarks[bindex, :, :] + return bbox, landmark + + @staticmethod + def check_large_pose(landmark, bbox): + assert landmark.shape == (5, 2) + assert len(bbox) == 4 + + def get_theta(base, x, y): + vx = x - base + vy = y - base + vx[1] *= -1 + vy[1] *= -1 + tx = np.arctan2(vx[1], vx[0]) + ty = np.arctan2(vy[1], vy[0]) + d = ty - tx + d = np.degrees(d) + #print(vx, tx, vy, ty, d) + #if d<-1.*math.pi: + # d+=2*math.pi + #elif d>math.pi: + # d-=2*math.pi + if d < -180.0: + d += 360. + elif d > 180.0: + d -= 360.0 + return d + + landmark = landmark.astype(np.float32) + + theta1 = get_theta(landmark[0], landmark[3], landmark[2]) + theta2 = get_theta(landmark[1], landmark[2], landmark[4]) + #print(va, vb, theta2) + theta3 = get_theta(landmark[0], landmark[2], landmark[1]) + theta4 = get_theta(landmark[1], landmark[0], landmark[2]) + theta5 = get_theta(landmark[3], landmark[4], landmark[2]) + theta6 = get_theta(landmark[4], landmark[2], landmark[3]) + theta7 = get_theta(landmark[3], landmark[2], landmark[0]) + theta8 = get_theta(landmark[4], landmark[1], landmark[2]) + #print(theta1, theta2, theta3, theta4, theta5, theta6, theta7, theta8) + left_score = 0.0 + right_score = 0.0 + up_score = 0.0 + down_score = 0.0 + if theta1 <= 0.0: + left_score = 10.0 + elif theta2 <= 0.0: + right_score = 10.0 + else: + left_score = theta2 / theta1 + right_score = theta1 / theta2 + if theta3 <= 10.0 or theta4 <= 10.0: + up_score = 10.0 + else: + up_score = max(theta1 / theta3, theta2 / theta4) + if theta5 <= 10.0 or theta6 <= 10.0: + down_score = 10.0 + else: + down_score = max(theta7 / theta5, theta8 / theta6) + mleft = (landmark[0][0] + landmark[3][0]) / 2 + mright = (landmark[1][0] + landmark[4][0]) / 2 + box_center = ((bbox[0] + bbox[2]) / 2, (bbox[1] + bbox[3]) / 2) + ret = 0 + if left_score >= 3.0: + ret = 1 + if ret == 0 and left_score >= 2.0: + if mright <= box_center[0]: + ret = 1 + if ret == 0 and right_score >= 3.0: + ret = 2 + if ret == 0 and right_score >= 2.0: + if mleft >= box_center[0]: + ret = 2 + if ret == 0 and up_score >= 2.0: + ret = 3 + if ret == 0 and down_score >= 5.0: + ret = 4 + return ret, left_score, right_score, up_score, down_score + + @staticmethod + def _filter_boxes(boxes, min_size): + """ Remove all boxes with any side smaller than min_size """ + ws = boxes[:, 2] - boxes[:, 0] + 1 + hs = boxes[:, 3] - boxes[:, 1] + 1 + keep = np.where((ws >= min_size) & (hs >= min_size))[0] + return keep + + @staticmethod + def _filter_boxes2(boxes, max_size, min_size): + """ Remove all boxes with any side smaller than min_size """ + ws = boxes[:, 2] - boxes[:, 0] + 1 + hs = boxes[:, 3] - boxes[:, 1] + 1 + if max_size > 0: + keep = np.where(np.minimum(ws, hs) < max_size)[0] + elif min_size > 0: + keep = np.where(np.maximum(ws, hs) > min_size)[0] + return keep + + @staticmethod + def _clip_pad(tensor, pad_shape): + """ + Clip boxes of the pad area. + :param tensor: [n, c, H, W] + :param pad_shape: [h, w] + :return: [n, c, h, w] + """ + H, W = tensor.shape[2:] + h, w = pad_shape + + if h < H or w < W: + tensor = tensor[:, :, :h, :w].copy() + + return tensor + + @staticmethod + def bbox_pred(boxes, box_deltas): + """ + Transform the set of class-agnostic boxes into class-specific boxes + by applying the predicted offsets (box_deltas) + :param boxes: !important [N 4] + :param box_deltas: [N, 4 * num_classes] + :return: [N 4 * num_classes] + """ + if boxes.shape[0] == 0: + return np.zeros((0, box_deltas.shape[1])) + + boxes = boxes.astype(np.float, copy=False) + widths = boxes[:, 2] - boxes[:, 0] + 1.0 + heights = boxes[:, 3] - boxes[:, 1] + 1.0 + ctr_x = boxes[:, 0] + 0.5 * (widths - 1.0) + ctr_y = boxes[:, 1] + 0.5 * (heights - 1.0) + + dx = box_deltas[:, 0:1] + dy = box_deltas[:, 1:2] + dw = box_deltas[:, 2:3] + dh = box_deltas[:, 3:4] + + pred_ctr_x = dx * widths[:, np.newaxis] + ctr_x[:, np.newaxis] + pred_ctr_y = dy * heights[:, np.newaxis] + ctr_y[:, np.newaxis] + pred_w = np.exp(dw) * widths[:, np.newaxis] + pred_h = np.exp(dh) * heights[:, np.newaxis] + + pred_boxes = np.zeros(box_deltas.shape) + # x1 + pred_boxes[:, 0:1] = pred_ctr_x - 0.5 * (pred_w - 1.0) + # y1 + pred_boxes[:, 1:2] = pred_ctr_y - 0.5 * (pred_h - 1.0) + # x2 + pred_boxes[:, 2:3] = pred_ctr_x + 0.5 * (pred_w - 1.0) + # y2 + pred_boxes[:, 3:4] = pred_ctr_y + 0.5 * (pred_h - 1.0) + + if box_deltas.shape[1] > 4: + pred_boxes[:, 4:] = box_deltas[:, 4:] + + return pred_boxes + + @staticmethod + def landmark_pred(boxes, landmark_deltas): + if boxes.shape[0] == 0: + return np.zeros((0, landmark_deltas.shape[1])) + boxes = boxes.astype(np.float, copy=False) + widths = boxes[:, 2] - boxes[:, 0] + 1.0 + heights = boxes[:, 3] - boxes[:, 1] + 1.0 + ctr_x = boxes[:, 0] + 0.5 * (widths - 1.0) + ctr_y = boxes[:, 1] + 0.5 * (heights - 1.0) + pred = landmark_deltas.copy() + for i in range(5): + pred[:, i, 0] = landmark_deltas[:, i, 0] * widths + ctr_x + pred[:, i, 1] = landmark_deltas[:, i, 1] * heights + ctr_y + return pred + #preds = [] + #for i in range(landmark_deltas.shape[1]): + # if i%2==0: + # pred = (landmark_deltas[:,i]*widths + ctr_x) + # else: + # pred = (landmark_deltas[:,i]*heights + ctr_y) + # preds.append(pred) + #preds = np.vstack(preds).transpose() + #return preds + + def bbox_vote(self, det): + #order = det[:, 4].ravel().argsort()[::-1] + #det = det[order, :] + if det.shape[0] == 0: + return np.zeros((0, 5)) + #dets = np.array([[10, 10, 20, 20, 0.002]]) + #det = np.empty(shape=[0, 5]) + dets = None + while det.shape[0] > 0: + if dets is not None and dets.shape[0] >= 750: + break + # IOU + area = (det[:, 2] - det[:, 0] + 1) * (det[:, 3] - det[:, 1] + 1) + xx1 = np.maximum(det[0, 0], det[:, 0]) + yy1 = np.maximum(det[0, 1], det[:, 1]) + xx2 = np.minimum(det[0, 2], det[:, 2]) + yy2 = np.minimum(det[0, 3], det[:, 3]) + w = np.maximum(0.0, xx2 - xx1 + 1) + h = np.maximum(0.0, yy2 - yy1 + 1) + inter = w * h + o = inter / (area[0] + area[:] - inter) + + # nms + merge_index = np.where(o >= self.nms_threshold)[0] + det_accu = det[merge_index, :] + det = np.delete(det, merge_index, 0) + if merge_index.shape[0] <= 1: + if det.shape[0] == 0: + try: + dets = np.row_stack((dets, det_accu)) + except: + dets = det_accu + continue + det_accu[:, + 0:4] = det_accu[:, 0:4] * np.tile(det_accu[:, -1:], + (1, 4)) + max_score = np.max(det_accu[:, 4]) + det_accu_sum = np.zeros((1, 5)) + det_accu_sum[:, 0:4] = np.sum(det_accu[:, 0:4], axis=0) / np.sum( + det_accu[:, -1:]) + det_accu_sum[:, 4] = max_score + if dets is None: + dets = det_accu_sum + else: + dets = np.row_stack((dets, det_accu_sum)) + dets = dets[0:750, :] + return dets diff --git a/detection/RetinaFace/test.py b/detection/RetinaFace/test.py new file mode 100644 index 0000000..b88c82b --- /dev/null +++ b/detection/RetinaFace/test.py @@ -0,0 +1,63 @@ +import cv2 +import sys +import numpy as np +import datetime +import os +import glob +from retinaface import RetinaFace + +thresh = 0.8 +scales = [1024, 1980] + +count = 1 + +gpuid = 0 +detector = RetinaFace('./model/R50', 0, gpuid, 'net3') + +img = cv2.imread('t1.jpg') +print(img.shape) +im_shape = img.shape +target_size = scales[0] +max_size = scales[1] +im_size_min = np.min(im_shape[0:2]) +im_size_max = np.max(im_shape[0:2]) +#im_scale = 1.0 +#if im_size_min>target_size or im_size_max>max_size: +im_scale = float(target_size) / float(im_size_min) +# prevent bigger axis from being more than max_size: +if np.round(im_scale * im_size_max) > max_size: + im_scale = float(max_size) / float(im_size_max) + +print('im_scale', im_scale) + +scales = [im_scale] +flip = False + +for c in range(count): + faces, landmarks = detector.detect(img, + thresh, + scales=scales, + do_flip=flip) + print(c, faces.shape, landmarks.shape) + +if faces is not None: + print('find', faces.shape[0], 'faces') + for i in range(faces.shape[0]): + #print('score', faces[i][4]) + box = faces[i].astype(np.int) + #color = (255,0,0) + color = (0, 0, 255) + cv2.rectangle(img, (box[0], box[1]), (box[2], box[3]), color, 2) + if landmarks is not None: + landmark5 = landmarks[i].astype(np.int) + #print(landmark.shape) + for l in range(landmark5.shape[0]): + color = (0, 0, 255) + if l == 0 or l == 3: + color = (0, 255, 0) + cv2.circle(img, (landmark5[l][0], landmark5[l][1]), 1, color, + 2) + + filename = './detector_test.jpg' + print('writing', filename) + cv2.imwrite(filename, img) diff --git a/detection/RetinaFace/test_widerface.py b/detection/RetinaFace/test_widerface.py new file mode 100644 index 0000000..78c2f83 --- /dev/null +++ b/detection/RetinaFace/test_widerface.py @@ -0,0 +1,259 @@ +from __future__ import print_function + +import argparse +import sys +import os +import time +import numpy as np +import mxnet as mx +from mxnet import ndarray as nd +import cv2 +from rcnn.logger import logger +#from rcnn.config import config, default, generate_config +#from rcnn.tools.test_rcnn import test_rcnn +#from rcnn.tools.test_rpn import test_rpn +from rcnn.processing.bbox_transform import nonlinear_pred, clip_boxes, landmark_pred +from rcnn.processing.generate_anchor import generate_anchors_fpn, anchors_plane +from rcnn.processing.nms import gpu_nms_wrapper +from rcnn.processing.bbox_transform import bbox_overlaps +from rcnn.dataset import retinaface +from retinaface import RetinaFace + + +def parse_args(): + parser = argparse.ArgumentParser( + description='Test widerface by retinaface detector') + # general + parser.add_argument('--network', + help='network name', + default='net3', + type=str) + parser.add_argument('--dataset', + help='dataset name', + default='retinaface', + type=str) + parser.add_argument('--image-set', + help='image_set name', + default='val', + type=str) + parser.add_argument('--root-path', + help='output data folder', + default='./data', + type=str) + parser.add_argument('--dataset-path', + help='dataset path', + default='./data/retinaface', + type=str) + parser.add_argument('--gpu', + help='GPU device to test with', + default=0, + type=int) + # testing + parser.add_argument('--prefix', + help='model to test with', + default='', + type=str) + parser.add_argument('--epoch', + help='model to test with', + default=0, + type=int) + parser.add_argument('--output', + help='output folder', + default='./wout', + type=str) + parser.add_argument('--nocrop', help='', action='store_true') + parser.add_argument('--thresh', + help='valid detection threshold', + default=0.02, + type=float) + parser.add_argument('--mode', + help='test mode, 0 for fast, 1 for accurate', + default=1, + type=int) + #parser.add_argument('--pyramid', help='enable pyramid test', action='store_true') + #parser.add_argument('--bbox-vote', help='', action='store_true') + parser.add_argument('--part', help='', default=0, type=int) + parser.add_argument('--parts', help='', default=1, type=int) + args = parser.parse_args() + return args + + +detector = None +args = None +imgid = -1 + + +def get_boxes(roi, pyramid): + global imgid + im = cv2.imread(roi['image']) + do_flip = False + if not pyramid: + target_size = 1200 + max_size = 1600 + #do_flip = True + target_size = 1504 + max_size = 2000 + target_size = 1600 + max_size = 2150 + im_shape = im.shape + im_size_min = np.min(im_shape[0:2]) + im_size_max = np.max(im_shape[0:2]) + im_scale = float(target_size) / float(im_size_min) + # prevent bigger axis from being more than max_size: + if np.round(im_scale * im_size_max) > max_size: + im_scale = float(max_size) / float(im_size_max) + scales = [im_scale] + else: + do_flip = True + #TEST_SCALES = [500, 800, 1200, 1600] + TEST_SCALES = [500, 800, 1100, 1400, 1700] + target_size = 800 + max_size = 1200 + im_shape = im.shape + im_size_min = np.min(im_shape[0:2]) + im_size_max = np.max(im_shape[0:2]) + im_scale = float(target_size) / float(im_size_min) + # prevent bigger axis from being more than max_size: + if np.round(im_scale * im_size_max) > max_size: + im_scale = float(max_size) / float(im_size_max) + scales = [ + float(scale) / target_size * im_scale for scale in TEST_SCALES + ] + boxes, landmarks = detector.detect(im, + threshold=args.thresh, + scales=scales, + do_flip=do_flip) + #print(boxes.shape, landmarks.shape) + if imgid >= 0 and imgid < 100: + font = cv2.FONT_HERSHEY_SIMPLEX + for i in range(boxes.shape[0]): + box = boxes[i] + ibox = box[0:4].copy().astype(np.int) + cv2.rectangle(im, (ibox[0], ibox[1]), (ibox[2], ibox[3]), + (255, 0, 0), 2) + #print('box', ibox) + #if len(ibox)>5: + # for l in range(5): + # pp = (ibox[5+l*2], ibox[6+l*2]) + # cv2.circle(im, (pp[0], pp[1]), 1, (0, 0, 255), 1) + blur = box[5] + k = "%.3f" % blur + cv2.putText(im, k, (ibox[0] + 2, ibox[1] + 14), font, 0.6, + (0, 255, 0), 2) + #landmarks = box[6:21].reshape( (5,3) ) + if landmarks is not None: + for l in range(5): + color = (0, 255, 0) + landmark = landmarks[i][l] + pp = (int(landmark[0]), int(landmark[1])) + if landmark[2] - 0.5 < 0.0: + color = (0, 0, 255) + cv2.circle(im, (pp[0], pp[1]), 1, color, 2) + filename = './testimages/%d.jpg' % imgid + cv2.imwrite(filename, im) + print(filename, 'wrote') + imgid += 1 + + return boxes + + +def test(args): + print('test with', args) + global detector + output_folder = args.output + if not os.path.exists(output_folder): + os.mkdir(output_folder) + detector = RetinaFace(args.prefix, + args.epoch, + args.gpu, + network=args.network, + nocrop=args.nocrop, + vote=args.bbox_vote) + imdb = eval(args.dataset)(args.image_set, args.root_path, + args.dataset_path) + roidb = imdb.gt_roidb() + gt_overlaps = np.zeros(0) + overall = [0.0, 0.0] + gt_max = np.array((0.0, 0.0)) + num_pos = 0 + print('roidb size', len(roidb)) + + for i in range(len(roidb)): + if i % args.parts != args.part: + continue + #if i%10==0: + # print('processing', i, file=sys.stderr) + roi = roidb[i] + boxes = get_boxes(roi, args.pyramid) + if 'boxes' in roi: + gt_boxes = roi['boxes'].copy() + gt_areas = (gt_boxes[:, 2] - gt_boxes[:, 0] + + 1) * (gt_boxes[:, 3] - gt_boxes[:, 1] + 1) + num_pos += gt_boxes.shape[0] + + overlaps = bbox_overlaps(boxes.astype(np.float), + gt_boxes.astype(np.float)) + #print(im_info, gt_boxes.shape, boxes.shape, overlaps.shape, file=sys.stderr) + + _gt_overlaps = np.zeros((gt_boxes.shape[0])) + + if boxes.shape[0] > 0: + _gt_overlaps = overlaps.max(axis=0) + #print('max_overlaps', _gt_overlaps, file=sys.stderr) + for j in range(len(_gt_overlaps)): + if _gt_overlaps[j] > 0.5: + continue + #print(j, 'failed', gt_boxes[j], 'max_overlap:', _gt_overlaps[j], file=sys.stderr) + + # append recorded IoU coverage level + found = (_gt_overlaps > 0.5).sum() + recall = found / float(gt_boxes.shape[0]) + #print('recall', _recall, gt_boxes.shape[0], boxes.shape[0], gt_areas, 'num:', i, file=sys.stderr) + overall[0] += found + overall[1] += gt_boxes.shape[0] + #gt_overlaps = np.hstack((gt_overlaps, _gt_overlaps)) + #_recall = (gt_overlaps >= threshold).sum() / float(num_pos) + recall_all = float(overall[0]) / overall[1] + #print('recall_all', _recall, file=sys.stderr) + print('[%d]' % i, + 'recall', + recall, (gt_boxes.shape[0], boxes.shape[0]), + 'all:', + recall_all, + file=sys.stderr) + else: + print('[%d]' % i, 'detect %d faces' % boxes.shape[0]) + + _vec = roidb[i]['image'].split('/') + out_dir = os.path.join(output_folder, _vec[-2]) + if not os.path.exists(out_dir): + os.mkdir(out_dir) + out_file = os.path.join(out_dir, _vec[-1].replace('jpg', 'txt')) + with open(out_file, 'w') as f: + name = '/'.join(roidb[i]['image'].split('/')[-2:]) + f.write("%s\n" % (name)) + f.write("%d\n" % (boxes.shape[0])) + for b in range(boxes.shape[0]): + box = boxes[b] + f.write( + "%d %d %d %d %g \n" % + (box[0], box[1], box[2] - box[0], box[3] - box[1], box[4])) + + +def main(): + global args + args = parse_args() + args.pyramid = False + args.bbox_vote = False + if args.mode == 1: + args.pyramid = True + args.bbox_vote = True + elif args.mode == 2: + args.pyramid = True + args.bbox_vote = False + logger.info('Called with argument: %s' % args) + test(args) + + +if __name__ == '__main__': + main() diff --git a/detection/RetinaFace/train.py b/detection/RetinaFace/train.py new file mode 100644 index 0000000..5fbcc70 --- /dev/null +++ b/detection/RetinaFace/train.py @@ -0,0 +1,502 @@ +from __future__ import print_function +import sys +import argparse +import os +import pprint +import re +import mxnet as mx +import numpy as np +from mxnet.module import Module +import mxnet.optimizer as optimizer + +from rcnn.logger import logger +from rcnn.config import config, default, generate_config +from rcnn.symbol import * +from rcnn.core import callback, metric +from rcnn.core.loader import CropLoader, CropLoader2 +from rcnn.core.module import MutableModule +from rcnn.utils.load_data import load_gt_roidb, merge_roidb, filter_roidb +from rcnn.utils.load_model import load_param + + +def get_fixed_params(symbol, fixed_param): + if not config.LAYER_FIX: + return [] + fixed_param_names = [] + #for name in symbol.list_arguments(): + # for f in fixed_param: + # if re.match(f, name): + # fixed_param_names.append(name) + #pre = 'mobilenetv20_features_linearbottleneck' + idx = 0 + for name in symbol.list_arguments(): + #print(idx, name) + if idx < 7 and name != 'data': + fixed_param_names.append(name) + #elif name.startswith('stage1_'): + # fixed_param_names.append(name) + if name.find('upsampling') >= 0: + fixed_param_names.append(name) + + idx += 1 + return fixed_param_names + + +def train_net(args, + ctx, + pretrained, + epoch, + prefix, + begin_epoch, + end_epoch, + lr=0.001, + lr_step='5'): + # setup config + #init_config() + #print(config) + # setup multi-gpu + + input_batch_size = config.TRAIN.BATCH_IMAGES * len(ctx) + + # print config + logger.info(pprint.pformat(config)) + + # load dataset and prepare imdb for training + image_sets = [iset for iset in args.image_set.split('+')] + roidbs = [ + load_gt_roidb(args.dataset, + image_set, + args.root_path, + args.dataset_path, + flip=not args.no_flip) for image_set in image_sets + ] + #roidb = merge_roidb(roidbs) + #roidb = filter_roidb(roidb) + roidb = roidbs[0] + + # load symbol + #sym = eval('get_' + args.network + '_train')(num_classes=config.NUM_CLASSES, num_anchors=config.NUM_ANCHORS) + #feat_sym = sym.get_internals()['rpn_cls_score_output'] + #train_data = AnchorLoader(feat_sym, roidb, batch_size=input_batch_size, shuffle=not args.no_shuffle, + # ctx=ctx, work_load_list=args.work_load_list, + # feat_stride=config.RPN_FEAT_STRIDE, anchor_scales=config.ANCHOR_SCALES, + # anchor_ratios=config.ANCHOR_RATIOS, aspect_grouping=config.TRAIN.ASPECT_GROUPING) + + # load and initialize params + sym = None + if len(pretrained) == 0: + arg_params = {} + aux_params = {} + else: + logger.info('loading %s,%d' % (pretrained, epoch)) + sym, arg_params, aux_params = mx.model.load_checkpoint( + pretrained, epoch) + #arg_params, aux_params = load_param(pretrained, epoch, convert=True) + #for k in ['rpn_conv_3x3', 'rpn_cls_score', 'rpn_bbox_pred', 'cls_score', 'bbox_pred']: + # _k = k+"_weight" + # if _k in arg_shape_dict: + # v = 0.001 if _k.startswith('bbox_') else 0.01 + # arg_params[_k] = mx.random.normal(0, v, shape=arg_shape_dict[_k]) + # print('init %s with normal %.5f'%(_k,v)) + # _k = k+"_bias" + # if _k in arg_shape_dict: + # arg_params[_k] = mx.nd.zeros(shape=arg_shape_dict[_k]) + # print('init %s with zero'%(_k)) + + sym = eval('get_' + args.network + '_train')(sym) + #print(sym.get_internals()) + feat_sym = [] + for stride in config.RPN_FEAT_STRIDE: + feat_sym.append( + sym.get_internals()['face_rpn_cls_score_stride%s_output' % stride]) + + train_data = CropLoader(feat_sym, + roidb, + batch_size=input_batch_size, + shuffle=not args.no_shuffle, + ctx=ctx, + work_load_list=args.work_load_list) + + # infer max shape + max_data_shape = [('data', (1, 3, max([v[1] for v in config.SCALES]), + max([v[1] for v in config.SCALES])))] + #max_data_shape = [('data', (1, 3, max([v[1] for v in config.SCALES]), max([v[1] for v in config.SCALES])))] + max_data_shape, max_label_shape = train_data.infer_shape(max_data_shape) + max_data_shape.append(('gt_boxes', (1, roidb[0]['max_num_boxes'], 5))) + logger.info('providing maximum shape %s %s' % + (max_data_shape, max_label_shape)) + + # infer shape + data_shape_dict = dict(train_data.provide_data + train_data.provide_label) + arg_shape, out_shape, aux_shape = sym.infer_shape(**data_shape_dict) + arg_shape_dict = dict(zip(sym.list_arguments(), arg_shape)) + out_shape_dict = dict(zip(sym.list_outputs(), out_shape)) + aux_shape_dict = dict(zip(sym.list_auxiliary_states(), aux_shape)) + logger.info('output shape %s' % pprint.pformat(out_shape_dict)) + + for k in arg_shape_dict: + v = arg_shape_dict[k] + if k.find('upsampling') >= 0: + print('initializing upsampling_weight', k) + arg_params[k] = mx.nd.zeros(shape=v) + init = mx.init.Initializer() + init._init_bilinear(k, arg_params[k]) + #print(args[k]) + + # check parameter shapes + #for k in sym.list_arguments(): + # if k in data_shape_dict: + # continue + # assert k in arg_params, k + ' not initialized' + # assert arg_params[k].shape == arg_shape_dict[k], \ + # 'shape inconsistent for ' + k + ' inferred ' + str(arg_shape_dict[k]) + ' provided ' + str(arg_params[k].shape) + #for k in sym.list_auxiliary_states(): + # assert k in aux_params, k + ' not initialized' + # assert aux_params[k].shape == aux_shape_dict[k], \ + # 'shape inconsistent for ' + k + ' inferred ' + str(aux_shape_dict[k]) + ' provided ' + str(aux_params[k].shape) + + fixed_param_prefix = config.FIXED_PARAMS + # create solver + data_names = [k[0] for k in train_data.provide_data] + label_names = [k[0] for k in train_data.provide_label] + fixed_param_names = get_fixed_params(sym, fixed_param_prefix) + print('fixed', fixed_param_names, file=sys.stderr) + mod = Module(sym, + data_names=data_names, + label_names=label_names, + logger=logger, + context=ctx, + work_load_list=args.work_load_list, + fixed_param_names=fixed_param_names) + + # metric + eval_metrics = mx.metric.CompositeEvalMetric() + mid = 0 + for m in range(len(config.RPN_FEAT_STRIDE)): + stride = config.RPN_FEAT_STRIDE[m] + #mid = m*MSTEP + _metric = metric.RPNAccMetric(pred_idx=mid, + label_idx=mid + 1, + name='RPNAcc_s%s' % stride) + eval_metrics.add(_metric) + mid += 2 + #_metric = metric.RPNLogLossMetric(pred_idx=mid, label_idx=mid+1) + #eval_metrics.add(_metric) + + _metric = metric.RPNL1LossMetric(loss_idx=mid, + weight_idx=mid + 1, + name='RPNL1Loss_s%s' % stride) + eval_metrics.add(_metric) + mid += 2 + if config.FACE_LANDMARK: + _metric = metric.RPNL1LossMetric(loss_idx=mid, + weight_idx=mid + 1, + name='RPNLandMarkL1Loss_s%s' % + stride) + eval_metrics.add(_metric) + mid += 2 + if config.HEAD_BOX: + _metric = metric.RPNAccMetric(pred_idx=mid, + label_idx=mid + 1, + name='RPNAcc_head_s%s' % stride) + eval_metrics.add(_metric) + mid += 2 + #_metric = metric.RPNLogLossMetric(pred_idx=mid, label_idx=mid+1) + #eval_metrics.add(_metric) + + _metric = metric.RPNL1LossMetric(loss_idx=mid, + weight_idx=mid + 1, + name='RPNL1Loss_head_s%s' % + stride) + eval_metrics.add(_metric) + mid += 2 + if config.CASCADE > 0: + for _idx in range(config.CASCADE): + if stride in config.CASCADE_CLS_STRIDES: + _metric = metric.RPNAccMetric(pred_idx=mid, + label_idx=mid + 1, + name='RPNAccCAS%d_s%s' % + (_idx, stride)) + eval_metrics.add(_metric) + mid += 2 + if stride in config.CASCADE_BBOX_STRIDES: + _metric = metric.RPNL1LossMetric( + loss_idx=mid, + weight_idx=mid + 1, + name='RPNL1LossCAS%d_s%s' % (_idx, stride)) + eval_metrics.add(_metric) + mid += 2 + + # callback + #means = np.tile(np.array(config.TRAIN.BBOX_MEANS), config.NUM_CLASSES) + #stds = np.tile(np.array(config.TRAIN.BBOX_STDS), config.NUM_CLASSES) + #epoch_end_callback = callback.do_checkpoint(prefix, means, stds) + epoch_end_callback = None + # decide learning rate + #base_lr = lr + #lr_factor = 0.1 + #lr = base_lr * (lr_factor ** (len(lr_epoch) - len(lr_epoch_diff))) + + lr_epoch = [int(epoch) for epoch in lr_step.split(',')] + lr_epoch_diff = [ + epoch - begin_epoch for epoch in lr_epoch if epoch > begin_epoch + ] + lr_iters = [ + int(epoch * len(roidb) / input_batch_size) for epoch in lr_epoch_diff + ] + iter_per_epoch = int(len(roidb) / input_batch_size) + + lr_steps = [] + if len(lr_iters) == 5: + factors = [0.5, 0.5, 0.4, 0.1, 0.1] + for i in range(5): + lr_steps.append((lr_iters[i], factors[i])) + elif len(lr_iters) == 8: #warmup + for li in lr_iters[0:5]: + lr_steps.append((li, 1.5849)) + for li in lr_iters[5:]: + lr_steps.append((li, 0.1)) + else: + for li in lr_iters: + lr_steps.append((li, 0.1)) + #lr_steps = [ (10,0.1), (20, 0.1) ] #XXX + + end_epoch = 10000 + logger.info('lr %f lr_epoch_diff %s lr_steps %s' % + (lr, lr_epoch_diff, lr_steps)) + # optimizer + opt = optimizer.SGD(learning_rate=lr, + momentum=0.9, + wd=args.wd, + rescale_grad=1.0 / len(ctx), + clip_gradient=None) + initializer = mx.init.Xavier() + #initializer = mx.init.Xavier(rnd_type='gaussian', factor_type="out", magnitude=2) #resnet style + + train_data = mx.io.PrefetchingIter(train_data) + + _cb = mx.callback.Speedometer(train_data.batch_size, + frequent=args.frequent, + auto_reset=False) + global_step = [0] + + def save_model(epoch): + arg, aux = mod.get_params() + all_layers = mod.symbol.get_internals() + outs = [] + for stride in config.RPN_FEAT_STRIDE: + num_anchors = config.RPN_ANCHOR_CFG[str(stride)]['NUM_ANCHORS'] + if config.CASCADE > 0: + _name = 'face_rpn_cls_score_stride%d_output' % (stride) + cls_pred = all_layers[_name] + cls_pred = mx.symbol.Reshape(data=cls_pred, + shape=(0, 2, -1, 0)) + + cls_pred = mx.symbol.SoftmaxActivation(data=cls_pred, + mode="channel") + cls_pred = mx.symbol.Reshape(data=cls_pred, + shape=(0, 2 * num_anchors, -1, 0)) + outs.append(cls_pred) + _name = 'face_rpn_bbox_pred_stride%d_output' % stride + rpn_bbox_pred = all_layers[_name] + outs.append(rpn_bbox_pred) + if config.FACE_LANDMARK: + _name = 'face_rpn_landmark_pred_stride%d_output' % stride + rpn_landmark_pred = all_layers[_name] + outs.append(rpn_landmark_pred) + for casid in range(config.CASCADE): + if stride in config.CASCADE_CLS_STRIDES: + _name = 'face_rpn_cls_score_stride%d_cas%d_output' % ( + stride, casid) + cls_pred = all_layers[_name] + cls_pred = mx.symbol.Reshape(data=cls_pred, + shape=(0, 2, -1, 0)) + cls_pred = mx.symbol.SoftmaxActivation(data=cls_pred, + mode="channel") + cls_pred = mx.symbol.Reshape(data=cls_pred, + shape=(0, 2 * num_anchors, + -1, 0)) + outs.append(cls_pred) + if stride in config.CASCADE_BBOX_STRIDES: + _name = 'face_rpn_bbox_pred_stride%d_cas%d_output' % ( + stride, casid) + bbox_pred = all_layers[_name] + outs.append(bbox_pred) + else: + _name = 'face_rpn_cls_score_stride%d_output' % stride + rpn_cls_score = all_layers[_name] + + # prepare rpn data + rpn_cls_score_reshape = mx.symbol.Reshape( + data=rpn_cls_score, + shape=(0, 2, -1, 0), + name="face_rpn_cls_score_reshape_stride%d" % stride) + + rpn_cls_prob = mx.symbol.SoftmaxActivation( + data=rpn_cls_score_reshape, + mode="channel", + name="face_rpn_cls_prob_stride%d" % stride) + rpn_cls_prob_reshape = mx.symbol.Reshape( + data=rpn_cls_prob, + shape=(0, 2 * num_anchors, -1, 0), + name='face_rpn_cls_prob_reshape_stride%d' % stride) + _name = 'face_rpn_bbox_pred_stride%d_output' % stride + rpn_bbox_pred = all_layers[_name] + outs.append(rpn_cls_prob_reshape) + outs.append(rpn_bbox_pred) + if config.FACE_LANDMARK: + _name = 'face_rpn_landmark_pred_stride%d_output' % stride + rpn_landmark_pred = all_layers[_name] + outs.append(rpn_landmark_pred) + _sym = mx.sym.Group(outs) + mx.model.save_checkpoint(prefix, epoch, _sym, arg, aux) + + def _batch_callback(param): + #global global_step + _cb(param) + global_step[0] += 1 + mbatch = global_step[0] + for step in lr_steps: + if mbatch == step[0]: + opt.lr *= step[1] + print('lr change to', + opt.lr, + ' in batch', + mbatch, + file=sys.stderr) + break + + if mbatch % iter_per_epoch == 0: + print('saving checkpoint', mbatch, file=sys.stderr) + save_model(0) + if mbatch == lr_steps[-1][0]: + print('saving final checkpoint', mbatch, file=sys.stderr) + save_model(0) + #arg, aux = mod.get_params() + #mx.model.save_checkpoint(prefix, 99, mod.symbol, arg, aux) + sys.exit(0) + + # train + mod.fit(train_data, + eval_metric=eval_metrics, + epoch_end_callback=epoch_end_callback, + batch_end_callback=_batch_callback, + kvstore=args.kvstore, + optimizer=opt, + initializer=initializer, + allow_missing=True, + arg_params=arg_params, + aux_params=aux_params, + begin_epoch=begin_epoch, + num_epoch=end_epoch) + + +def parse_args(): + parser = argparse.ArgumentParser(description='Train RetinaFace') + # general + parser.add_argument('--network', + help='network name', + default=default.network, + type=str) + parser.add_argument('--dataset', + help='dataset name', + default=default.dataset, + type=str) + args, rest = parser.parse_known_args() + generate_config(args.network, args.dataset) + parser.add_argument('--image_set', + help='image_set name', + default=default.image_set, + type=str) + parser.add_argument('--root_path', + help='output data folder', + default=default.root_path, + type=str) + parser.add_argument('--dataset_path', + help='dataset path', + default=default.dataset_path, + type=str) + # training + parser.add_argument('--frequent', + help='frequency of logging', + default=default.frequent, + type=int) + parser.add_argument('--kvstore', + help='the kv-store type', + default=default.kvstore, + type=str) + parser.add_argument('--work_load_list', + help='work load for different devices', + default=None, + type=list) + parser.add_argument('--no_flip', + help='disable flip images', + action='store_true') + parser.add_argument('--no_shuffle', + help='disable random shuffle', + action='store_true') + # e2e + #parser.add_argument('--gpus', help='GPU device to train with', default='0,1,2,3', type=str) + parser.add_argument('--pretrained', + help='pretrained model prefix', + default=default.pretrained, + type=str) + parser.add_argument('--pretrained_epoch', + help='pretrained model epoch', + default=default.pretrained_epoch, + type=int) + parser.add_argument('--prefix', + help='new model prefix', + default=default.prefix, + type=str) + parser.add_argument('--begin_epoch', + help='begin epoch of training, use with resume', + default=0, + type=int) + parser.add_argument('--end_epoch', + help='end epoch of training', + default=default.end_epoch, + type=int) + parser.add_argument('--lr', + help='base learning rate', + default=default.lr, + type=float) + parser.add_argument('--lr_step', + help='learning rate steps (in epoch)', + default=default.lr_step, + type=str) + parser.add_argument('--wd', + help='weight decay', + default=default.wd, + type=float) + args = parser.parse_args() + return args + + +def main(): + args = parse_args() + logger.info('Called with argument: %s' % args) + #ctx = [mx.gpu(int(i)) for i in args.gpus.split(',')] + ctx = [] + cvd = os.environ['CUDA_VISIBLE_DEVICES'].strip() + if len(cvd) > 0: + for i in range(len(cvd.split(','))): + ctx.append(mx.gpu(i)) + if len(ctx) == 0: + ctx = [mx.cpu()] + print('use cpu') + else: + print('gpu num:', len(ctx)) + train_net(args, + ctx, + args.pretrained, + args.pretrained_epoch, + args.prefix, + args.begin_epoch, + args.end_epoch, + lr=args.lr, + lr_step=args.lr_step) + + +if __name__ == '__main__': + main() diff --git a/RetinaFaceAntiCov/README.md b/detection/RetinaFaceAntiCov/README.md similarity index 100% rename from RetinaFaceAntiCov/README.md rename to detection/RetinaFaceAntiCov/README.md diff --git a/RetinaFaceAntiCov/rcnn/processing/__init__.py b/detection/RetinaFaceAntiCov/rcnn/processing/__init__.py similarity index 100% rename from RetinaFaceAntiCov/rcnn/processing/__init__.py rename to detection/RetinaFaceAntiCov/rcnn/processing/__init__.py diff --git a/RetinaFace/rcnn/processing/assign_levels.py b/detection/RetinaFaceAntiCov/rcnn/processing/assign_levels.py similarity index 73% rename from RetinaFace/rcnn/processing/assign_levels.py rename to detection/RetinaFaceAntiCov/rcnn/processing/assign_levels.py index b237439..012d73d 100755 --- a/RetinaFace/rcnn/processing/assign_levels.py +++ b/detection/RetinaFaceAntiCov/rcnn/processing/assign_levels.py @@ -3,7 +3,8 @@ import numpy as np def compute_assign_targets(rois, threshold): - rois_area = np.sqrt((rois[:, 2] - rois[:, 0] + 1) * (rois[:, 3] - rois[:, 1] + 1)) + rois_area = np.sqrt( + (rois[:, 2] - rois[:, 0] + 1) * (rois[:, 3] - rois[:, 1] + 1)) num_rois = np.shape(rois)[0] assign_levels = np.zeros(num_rois, dtype=np.uint8) for i, stride in enumerate(config.RCNN_FEAT_STRIDE): @@ -24,14 +25,12 @@ def add_assign_targets(roidb): assert len(roidb) > 0 assert 'boxes' in roidb[0] - area_threshold = [[np.inf, 448], - [448, 224], - [224, 112], - [112, 0]] + area_threshold = [[np.inf, 448], [448, 224], [224, 112], [112, 0]] assert len(config.RCNN_FEAT_STRIDE) == len(area_threshold) num_images = len(roidb) for im_i in range(num_images): rois = roidb[im_i]['boxes'] - roidb[im_i]['assign_levels'] = compute_assign_targets(rois, area_threshold) + roidb[im_i]['assign_levels'] = compute_assign_targets( + rois, area_threshold) diff --git a/RetinaFace/rcnn/processing/bbox_regression.py b/detection/RetinaFaceAntiCov/rcnn/processing/bbox_regression.py similarity index 93% rename from RetinaFace/rcnn/processing/bbox_regression.py rename to detection/RetinaFaceAntiCov/rcnn/processing/bbox_regression.py index 11d1a02..0eaf917 100644 --- a/RetinaFace/rcnn/processing/bbox_regression.py +++ b/detection/RetinaFaceAntiCov/rcnn/processing/bbox_regression.py @@ -69,7 +69,8 @@ def add_bbox_regression_targets(roidb): rois = roidb[im_i]['boxes'] max_overlaps = roidb[im_i]['max_overlaps'] max_classes = roidb[im_i]['max_classes'] - roidb[im_i]['bbox_targets'] = compute_bbox_regression_targets(rois, max_overlaps, max_classes) + roidb[im_i]['bbox_targets'] = compute_bbox_regression_targets( + rois, max_overlaps, max_classes) if config.TRAIN.BBOX_NORMALIZATION_PRECOMPUTED: # use fixed / precomputed means and stds instead of empirical values @@ -87,11 +88,12 @@ def add_bbox_regression_targets(roidb): if cls_indexes.size > 0: class_counts[cls] += cls_indexes.size sums[cls, :] += targets[cls_indexes, 1:].sum(axis=0) - squared_sums[cls, :] += (targets[cls_indexes, 1:] ** 2).sum(axis=0) + squared_sums[cls, :] += (targets[cls_indexes, + 1:]**2).sum(axis=0) means = sums / class_counts # var(x) = E(x^2) - E(x)^2 - stds = np.sqrt(squared_sums / class_counts - means ** 2) + stds = np.sqrt(squared_sums / class_counts - means**2) # normalized targets for im_i in range(num_images): @@ -142,7 +144,8 @@ def compute_mask_and_label(ex_rois, ex_labels, seg, flipped): mask_target = np.zeros((n_rois, 28, 28), dtype=np.int8) mask_label = np.zeros((n_rois), dtype=np.int8) for n in range(n_rois): - target = ins_seg[int(rois[n, 1]): int(rois[n, 3]), int(rois[n, 0]): int(rois[n, 2])] + target = ins_seg[int(rois[n, 1]):int(rois[n, 3]), + int(rois[n, 0]):int(rois[n, 2])] ids = np.unique(target) ins_id = 0 max_count = 0 @@ -158,8 +161,9 @@ def compute_mask_and_label(ex_rois, ex_labels, seg, flipped): x2 = min(rois[n, 2], x_max) y2 = min(rois[n, 3], y_max) iou = (x2 - x1) * (y2 - y1) - iou = iou / ((rois[n, 2] - rois[n, 0]) * (rois[n, 3] - rois[n, 1]) - + (x_max - x_min) * (y_max - y_min) - iou) + iou = iou / ((rois[n, 2] - rois[n, 0]) * + (rois[n, 3] - rois[n, 1]) + (x_max - x_min) * + (y_max - y_min) - iou) if iou > max_count: ins_id = id max_count = iou @@ -202,16 +206,17 @@ def compute_bbox_mask_targets_and_label(rois, overlaps, labels, seg, flipped): # Get IoU overlap between each ex ROI and gt ROI ex_gt_overlaps = bbox_overlaps(rois[ex_inds, :], rois[gt_inds, :]) - # Find which gt ROI each ex ROI has max overlap with: # this will be the ex ROI's gt target gt_assignment = ex_gt_overlaps.argmax(axis=1) gt_rois = rois[gt_inds[gt_assignment], :] ex_rois = rois[ex_inds, :] - mask_targets, mask_label = compute_mask_and_label(ex_rois, labels[ex_inds], seg, flipped) + mask_targets, mask_label = compute_mask_and_label(ex_rois, labels[ex_inds], + seg, flipped) return mask_targets, mask_label, ex_inds + def add_mask_targets(roidb): """ given roidb, add ['bbox_targets'] and normalize bounding box regression targets @@ -240,9 +245,12 @@ def add_mask_targets(roidb): flipped = roidb[im_i]['flipped'] roidb[im_i]['mask_targets'], roidb[im_i]['mask_labels'], roidb[im_i]['mask_inds'] = \ compute_bbox_mask_targets_and_label(rois, max_overlaps, max_classes, ins_seg, flipped) + threads = [threading.Thread(target=process, args=()) for i in range(10)] - for t in threads: t.start() - for t in threads: t.join() + for t in threads: + t.start() + for t in threads: + t.join() # Single thread # for im_i in range(num_images): # print "-----processing img {}".format(im_i) diff --git a/RetinaFaceAntiCov/rcnn/processing/bbox_transform.py b/detection/RetinaFaceAntiCov/rcnn/processing/bbox_transform.py similarity index 79% rename from RetinaFaceAntiCov/rcnn/processing/bbox_transform.py rename to detection/RetinaFaceAntiCov/rcnn/processing/bbox_transform.py index 0c4fc7c..5ee3646 100644 --- a/RetinaFaceAntiCov/rcnn/processing/bbox_transform.py +++ b/detection/RetinaFaceAntiCov/rcnn/processing/bbox_transform.py @@ -18,13 +18,17 @@ def bbox_overlaps_py(boxes, query_boxes): k_ = query_boxes.shape[0] overlaps = np.zeros((n_, k_), dtype=np.float) for k in range(k_): - query_box_area = (query_boxes[k, 2] - query_boxes[k, 0] + 1) * (query_boxes[k, 3] - query_boxes[k, 1] + 1) + query_box_area = (query_boxes[k, 2] - query_boxes[k, 0] + + 1) * (query_boxes[k, 3] - query_boxes[k, 1] + 1) for n in range(n_): - iw = min(boxes[n, 2], query_boxes[k, 2]) - max(boxes[n, 0], query_boxes[k, 0]) + 1 + iw = min(boxes[n, 2], query_boxes[k, 2]) - max( + boxes[n, 0], query_boxes[k, 0]) + 1 if iw > 0: - ih = min(boxes[n, 3], query_boxes[k, 3]) - max(boxes[n, 1], query_boxes[k, 1]) + 1 + ih = min(boxes[n, 3], query_boxes[k, 3]) - max( + boxes[n, 1], query_boxes[k, 1]) + 1 if ih > 0: - box_area = (boxes[n, 2] - boxes[n, 0] + 1) * (boxes[n, 3] - boxes[n, 1] + 1) + box_area = (boxes[n, 2] - boxes[n, 0] + + 1) * (boxes[n, 3] - boxes[n, 1] + 1) all_area = float(box_area + query_box_area - iw * ih) overlaps[n, k] = iw * ih / all_area return overlaps @@ -72,18 +76,19 @@ def nonlinear_transform(ex_rois, gt_rois): targets_dw = np.log(gt_widths / ex_widths) targets_dh = np.log(gt_heights / ex_heights) - if gt_rois.shape[1]<=4: - targets = np.vstack( - (targets_dx, targets_dy, targets_dw, targets_dh)).transpose() - return targets + if gt_rois.shape[1] <= 4: + targets = np.vstack( + (targets_dx, targets_dy, targets_dw, targets_dh)).transpose() + return targets else: - targets = [targets_dx, targets_dy, targets_dw, targets_dh] - #if config.USE_BLUR: - # for i in range(4, gt_rois.shape[1]): - # t = gt_rois[:,i] - # targets.append(t) - targets = np.vstack(targets).transpose() - return targets + targets = [targets_dx, targets_dy, targets_dw, targets_dh] + #if config.USE_BLUR: + # for i in range(4, gt_rois.shape[1]): + # t = gt_rois[:,i] + # targets.append(t) + targets = np.vstack(targets).transpose() + return targets + def landmark_transform(ex_rois, gt_rois): @@ -94,22 +99,20 @@ def landmark_transform(ex_rois, gt_rois): ex_ctr_x = ex_rois[:, 0] + 0.5 * (ex_widths - 1.0) ex_ctr_y = ex_rois[:, 1] + 0.5 * (ex_heights - 1.0) - targets = [] for i in range(gt_rois.shape[1]): - for j in range(gt_rois.shape[2]): - #if not config.USE_OCCLUSION and j==2: - # continue - if j==2: - continue - if j==0: #w - target = (gt_rois[:,i,j] - ex_ctr_x) / (ex_widths + 1e-14) - elif j==1: #h - target = (gt_rois[:,i,j] - ex_ctr_y) / (ex_heights + 1e-14) - else: #visibile - target = gt_rois[:,i,j] - targets.append(target) - + for j in range(gt_rois.shape[2]): + #if not config.USE_OCCLUSION and j==2: + # continue + if j == 2: + continue + if j == 0: #w + target = (gt_rois[:, i, j] - ex_ctr_x) / (ex_widths + 1e-14) + elif j == 1: #h + target = (gt_rois[:, i, j] - ex_ctr_y) / (ex_heights + 1e-14) + else: #visibile + target = gt_rois[:, i, j] + targets.append(target) targets = np.vstack(targets).transpose() return targets @@ -154,6 +157,7 @@ def nonlinear_pred(boxes, box_deltas): return pred_boxes + def landmark_pred(boxes, landmark_deltas): if boxes.shape[0] == 0: return np.zeros((0, landmark_deltas.shape[1])) @@ -164,14 +168,15 @@ def landmark_pred(boxes, landmark_deltas): ctr_y = boxes[:, 1] + 0.5 * (heights - 1.0) preds = [] for i in range(landmark_deltas.shape[1]): - if i%2==0: - pred = (landmark_deltas[:,i]*widths + ctr_x) - else: - pred = (landmark_deltas[:,i]*heights + ctr_y) - preds.append(pred) + if i % 2 == 0: + pred = (landmark_deltas[:, i] * widths + ctr_x) + else: + pred = (landmark_deltas[:, i] * heights + ctr_y) + preds.append(pred) preds = np.vstack(preds).transpose() return preds + def iou_transform(ex_rois, gt_rois): """ return bbox targets, IoU loss uses gt_rois as gt """ assert ex_rois.shape[0] == gt_rois.shape[0], 'inconsistent rois number' diff --git a/RetinaFaceAntiCov/rcnn/processing/bbox_transform.py.orig b/detection/RetinaFaceAntiCov/rcnn/processing/bbox_transform.py.orig similarity index 100% rename from RetinaFaceAntiCov/rcnn/processing/bbox_transform.py.orig rename to detection/RetinaFaceAntiCov/rcnn/processing/bbox_transform.py.orig diff --git a/RetinaFace/rcnn/processing/generate_anchor.py b/detection/RetinaFaceAntiCov/rcnn/processing/generate_anchor.py similarity index 70% rename from RetinaFace/rcnn/processing/generate_anchor.py rename to detection/RetinaFaceAntiCov/rcnn/processing/generate_anchor.py index eb47d8e..83c5ada 100644 --- a/RetinaFace/rcnn/processing/generate_anchor.py +++ b/detection/RetinaFaceAntiCov/rcnn/processing/generate_anchor.py @@ -12,8 +12,12 @@ from ..cython.anchors import anchors_cython def anchors_plane(feat_h, feat_w, stride, base_anchor): return anchors_cython(feat_h, feat_w, stride, base_anchor) -def generate_anchors(base_size=16, ratios=[0.5, 1, 2], - scales=2 ** np.arange(3, 6), stride=16, dense_anchor=False): + +def generate_anchors(base_size=16, + ratios=[0.5, 1, 2], + scales=2**np.arange(3, 6), + stride=16, + dense_anchor=False): """ Generate anchor (reference) windows by enumerating aspect ratios X scales wrt a reference (0, 0, 15, 15) window. @@ -21,16 +25,19 @@ def generate_anchors(base_size=16, ratios=[0.5, 1, 2], base_anchor = np.array([1, 1, base_size, base_size]) - 1 ratio_anchors = _ratio_enum(base_anchor, ratios) - anchors = np.vstack([_scale_enum(ratio_anchors[i, :], scales) - for i in range(ratio_anchors.shape[0])]) + anchors = np.vstack([ + _scale_enum(ratio_anchors[i, :], scales) + for i in range(ratio_anchors.shape[0]) + ]) if dense_anchor: - assert stride%2==0 - anchors2 = anchors.copy() - anchors2[:,:] += int(stride/2) - anchors = np.vstack( (anchors, anchors2) ) + assert stride % 2 == 0 + anchors2 = anchors.copy() + anchors2[:, :] += int(stride / 2) + anchors = np.vstack((anchors, anchors2)) #print('GA',base_anchor.shape, ratio_anchors.shape, anchors.shape) return anchors + #def generate_anchors_fpn(base_size=[64,32,16,8,4], ratios=[0.5, 1, 2], scales=8): # """ # Generate anchor (reference) windows by enumerating aspect ratios X @@ -48,33 +55,35 @@ def generate_anchors(base_size=16, ratios=[0.5, 1, 2], # anchors.append(r) # return anchors -def generate_anchors_fpn(dense_anchor=False, cfg = None): + +def generate_anchors_fpn(dense_anchor=False, cfg=None): #assert(False) """ Generate anchor (reference) windows by enumerating aspect ratios X scales wrt a reference (0, 0, 15, 15) window. """ if cfg is None: - from ..config import config - cfg = config.RPN_ANCHOR_CFG + from ..config import config + cfg = config.RPN_ANCHOR_CFG RPN_FEAT_STRIDE = [] for k in cfg: - RPN_FEAT_STRIDE.append( int(k) ) + RPN_FEAT_STRIDE.append(int(k)) RPN_FEAT_STRIDE = sorted(RPN_FEAT_STRIDE, reverse=True) anchors = [] for k in RPN_FEAT_STRIDE: - v = cfg[str(k)] - bs = v['BASE_SIZE'] - __ratios = np.array(v['RATIOS']) - __scales = np.array(v['SCALES']) - stride = int(k) - #print('anchors_fpn', bs, __ratios, __scales, file=sys.stderr) - r = generate_anchors(bs, __ratios, __scales, stride, dense_anchor) - #print('anchors_fpn', r.shape, file=sys.stderr) - anchors.append(r) + v = cfg[str(k)] + bs = v['BASE_SIZE'] + __ratios = np.array(v['RATIOS']) + __scales = np.array(v['SCALES']) + stride = int(k) + #print('anchors_fpn', bs, __ratios, __scales, file=sys.stderr) + r = generate_anchors(bs, __ratios, __scales, stride, dense_anchor) + #print('anchors_fpn', r.shape, file=sys.stderr) + anchors.append(r) return anchors + def _whctrs(anchor): """ Return width, height, x center, and y center for an anchor (window). @@ -95,10 +104,8 @@ def _mkanchors(ws, hs, x_ctr, y_ctr): ws = ws[:, np.newaxis] hs = hs[:, np.newaxis] - anchors = np.hstack((x_ctr - 0.5 * (ws - 1), - y_ctr - 0.5 * (hs - 1), - x_ctr + 0.5 * (ws - 1), - y_ctr + 0.5 * (hs - 1))) + anchors = np.hstack((x_ctr - 0.5 * (ws - 1), y_ctr - 0.5 * (hs - 1), + x_ctr + 0.5 * (ws - 1), y_ctr + 0.5 * (hs - 1))) return anchors diff --git a/RetinaFaceAntiCov/rcnn/processing/nms.py b/detection/RetinaFaceAntiCov/rcnn/processing/nms.py similarity index 99% rename from RetinaFaceAntiCov/rcnn/processing/nms.py rename to detection/RetinaFaceAntiCov/rcnn/processing/nms.py index 230139c..b32d92d 100644 --- a/RetinaFaceAntiCov/rcnn/processing/nms.py +++ b/detection/RetinaFaceAntiCov/rcnn/processing/nms.py @@ -9,18 +9,21 @@ except ImportError: def py_nms_wrapper(thresh): def _nms(dets): return nms(dets, thresh) + return _nms def cpu_nms_wrapper(thresh): def _nms(dets): return cpu_nms(dets, thresh) + return _nms def gpu_nms_wrapper(thresh, device_id): def _nms(dets): return gpu_nms(dets, thresh, device_id) + if gpu_nms is not None: return _nms else: diff --git a/detection/RetinaFaceAntiCov/retinaface_cov.py b/detection/RetinaFaceAntiCov/retinaface_cov.py new file mode 100644 index 0000000..a4d1c1a --- /dev/null +++ b/detection/RetinaFaceAntiCov/retinaface_cov.py @@ -0,0 +1,752 @@ +from __future__ import print_function +import sys +import os +import datetime +import time +import numpy as np +import mxnet as mx +from mxnet import ndarray as nd +import cv2 +#from rcnn import config +#from rcnn.processing.bbox_transform import nonlinear_pred, clip_boxes, landmark_pred +from rcnn.processing.bbox_transform import clip_boxes +from rcnn.processing.generate_anchor import generate_anchors_fpn, anchors_plane +from rcnn.processing.nms import gpu_nms_wrapper, cpu_nms_wrapper +from rcnn.processing.bbox_transform import bbox_overlaps + + +class RetinaFaceCoV: + def __init__(self, + prefix, + epoch, + ctx_id=0, + network='net3', + nms=0.4, + nocrop=False): + self.ctx_id = ctx_id + self.network = network + self.nms_threshold = nms + self.nocrop = nocrop + self.debug = False + self.fpn_keys = [] + self.anchor_cfg = None + pixel_means = [0.0, 0.0, 0.0] + pixel_stds = [1.0, 1.0, 1.0] + pixel_scale = 1.0 + self.bbox_stds = [1.0, 1.0, 1.0, 1.0] + self.landmark_std = 1.0 + self.preprocess = False + _ratio = (1., ) + fmc = 3 + if network == 'ssh' or network == 'vgg': + pixel_means = [103.939, 116.779, 123.68] + self.preprocess = True + elif network == 'net3': + _ratio = (1., ) + elif network == 'net3l': + _ratio = (1., ) + self.landmark_std = 0.2 + elif network == 'net3a': + _ratio = (1., 1.5) + elif network == 'net6': #like pyramidbox or s3fd + fmc = 6 + elif network == 'net5': #retinaface + fmc = 5 + elif network == 'net5a': + fmc = 5 + _ratio = (1., 1.5) + elif network == 'net4': + fmc = 4 + elif network == 'net4a': + fmc = 4 + _ratio = (1., 1.5) + elif network == 'x5': + fmc = 5 + pixel_means = [103.52, 116.28, 123.675] + pixel_stds = [57.375, 57.12, 58.395] + elif network == 'x3': + fmc = 3 + pixel_means = [103.52, 116.28, 123.675] + pixel_stds = [57.375, 57.12, 58.395] + elif network == 'x3a': + fmc = 3 + _ratio = (1., 1.5) + pixel_means = [103.52, 116.28, 123.675] + pixel_stds = [57.375, 57.12, 58.395] + else: + assert False, 'network setting error %s' % network + + if fmc == 3: + self._feat_stride_fpn = [32, 16, 8] + self.anchor_cfg = { + '32': { + 'SCALES': (32, 16), + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + }, + '16': { + 'SCALES': (8, 4), + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + }, + '8': { + 'SCALES': (2, 1), + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + }, + } + elif fmc == 4: + self._feat_stride_fpn = [32, 16, 8, 4] + self.anchor_cfg = { + '32': { + 'SCALES': (32, 16), + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + }, + '16': { + 'SCALES': (8, 4), + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + }, + '8': { + 'SCALES': (2, 1), + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + }, + '4': { + 'SCALES': (2, 1), + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + }, + } + elif fmc == 6: + self._feat_stride_fpn = [128, 64, 32, 16, 8, 4] + self.anchor_cfg = { + '128': { + 'SCALES': (32, ), + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + }, + '64': { + 'SCALES': (16, ), + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + }, + '32': { + 'SCALES': (8, ), + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + }, + '16': { + 'SCALES': (4, ), + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + }, + '8': { + 'SCALES': (2, ), + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + }, + '4': { + 'SCALES': (1, ), + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + }, + } + elif fmc == 5: + self._feat_stride_fpn = [64, 32, 16, 8, 4] + self.anchor_cfg = {} + _ass = 2.0**(1.0 / 3) + _basescale = 1.0 + for _stride in [4, 8, 16, 32, 64]: + key = str(_stride) + value = { + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + } + scales = [] + for _ in range(3): + scales.append(_basescale) + _basescale *= _ass + value['SCALES'] = tuple(scales) + self.anchor_cfg[key] = value + + #print(self._feat_stride_fpn, self.anchor_cfg) + + for s in self._feat_stride_fpn: + self.fpn_keys.append('stride%s' % s) + + dense_anchor = False + #self._anchors_fpn = dict(zip(self.fpn_keys, generate_anchors_fpn(base_size=fpn_base_size, scales=self._scales, ratios=self._ratios))) + self._anchors_fpn = dict( + zip( + self.fpn_keys, + generate_anchors_fpn(dense_anchor=dense_anchor, + cfg=self.anchor_cfg))) + for k in self._anchors_fpn: + v = self._anchors_fpn[k].astype(np.float32) + self._anchors_fpn[k] = v + + self._num_anchors = dict( + zip(self.fpn_keys, + [anchors.shape[0] for anchors in self._anchors_fpn.values()])) + #self._bbox_pred = nonlinear_pred + #self._landmark_pred = landmark_pred + sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) + if self.ctx_id >= 0: + self.ctx = mx.gpu(self.ctx_id) + self.nms = gpu_nms_wrapper(self.nms_threshold, self.ctx_id) + else: + self.ctx = mx.cpu() + self.nms = cpu_nms_wrapper(self.nms_threshold) + self.pixel_means = np.array(pixel_means, dtype=np.float32) + self.pixel_stds = np.array(pixel_stds, dtype=np.float32) + self.pixel_scale = float(pixel_scale) + #print('means', self.pixel_means) + self.use_landmarks = True + #print('use_landmarks', self.use_landmarks) + self.cascade = 0 + + if self.debug: + c = len(sym) // len(self._feat_stride_fpn) + sym = sym[(c * 0):] + self._feat_stride_fpn = [32, 16, 8] + #print('sym size:', len(sym)) + + image_size = (640, 640) + self.model = mx.mod.Module(symbol=sym, + context=self.ctx, + label_names=None) + self.model.bind(data_shapes=[('data', (1, 3, image_size[0], + image_size[1]))], + for_training=False) + self.model.set_params(arg_params, aux_params) + + def get_input(self, img): + im = img.astype(np.float32) + im_tensor = np.zeros((1, 3, im.shape[0], im.shape[1])) + for i in range(3): + im_tensor[ + 0, + i, :, :] = (im[:, :, 2 - i] / self.pixel_scale - + self.pixel_means[2 - i]) / self.pixel_stds[2 - i] + #if self.debug: + # timeb = datetime.datetime.now() + # diff = timeb - timea + # print('X2 uses', diff.total_seconds(), 'seconds') + data = nd.array(im_tensor) + return data + + def detect(self, img, threshold=0.5, scales=[1.0], do_flip=False): + #print('in_detect', threshold, scales, do_flip, do_nms) + proposals_list = [] + scores_list = [] + mask_scores_list = [] + landmarks_list = [] + strides_list = [] + timea = datetime.datetime.now() + flips = [0] + if do_flip: + flips = [0, 1] + + imgs = [img] + if isinstance(img, list): + imgs = img + for img in imgs: + for im_scale in scales: + for flip in flips: + if im_scale != 1.0: + im = cv2.resize(img, + None, + None, + fx=im_scale, + fy=im_scale, + interpolation=cv2.INTER_LINEAR) + else: + im = img.copy() + if flip: + im = im[:, ::-1, :] + if self.nocrop: + if im.shape[0] % 32 == 0: + h = im.shape[0] + else: + h = (im.shape[0] // 32 + 1) * 32 + if im.shape[1] % 32 == 0: + w = im.shape[1] + else: + w = (im.shape[1] // 32 + 1) * 32 + _im = np.zeros((h, w, 3), dtype=np.float32) + _im[0:im.shape[0], 0:im.shape[1], :] = im + im = _im + else: + im = im.astype(np.float32) + if self.debug: + timeb = datetime.datetime.now() + diff = timeb - timea + print('X1 uses', diff.total_seconds(), 'seconds') + #self.model.bind(data_shapes=[('data', (1, 3, image_size[0], image_size[1]))], for_training=False) + #im_info = [im.shape[0], im.shape[1], im_scale] + im_info = [im.shape[0], im.shape[1]] + im_tensor = np.zeros((1, 3, im.shape[0], im.shape[1])) + for i in range(3): + im_tensor[0, i, :, :] = ( + im[:, :, 2 - i] / self.pixel_scale - + self.pixel_means[2 - i]) / self.pixel_stds[2 - i] + if self.debug: + timeb = datetime.datetime.now() + diff = timeb - timea + print('X2 uses', diff.total_seconds(), 'seconds') + data = nd.array(im_tensor) + db = mx.io.DataBatch(data=(data, ), + provide_data=[('data', data.shape)]) + if self.debug: + timeb = datetime.datetime.now() + diff = timeb - timea + print('X3 uses', diff.total_seconds(), 'seconds') + self.model.forward(db, is_train=False) + net_out = self.model.get_outputs() + #post_nms_topN = self._rpn_post_nms_top_n + #min_size_dict = self._rpn_min_size_fpn + + sym_idx = 0 + + for _idx, s in enumerate(self._feat_stride_fpn): + #if len(scales)>1 and s==32 and im_scale==scales[-1]: + # continue + _key = 'stride%s' % s + stride = int(s) + is_cascade = False + #if self.vote and stride==4 and len(scales)>2 and (im_scale==scales[0]): + # continue + #print('getting', im_scale, stride, idx, len(net_out), data.shape, file=sys.stderr) + scores = net_out[sym_idx].asnumpy() + type_scores = net_out[sym_idx + 3].asnumpy() + print(scores.shape, type_scores.shape) + if self.debug: + timeb = datetime.datetime.now() + diff = timeb - timea + print('A uses', diff.total_seconds(), 'seconds') + A = self._num_anchors['stride%s' % s] + #print(scores.shape) + #print('scores',stride, scores.shape, file=sys.stderr) + scores = scores[:, A:, :, :] + mask_scores = type_scores[:, A * 2:, :, :] #x, A, x, x + + bbox_deltas = net_out[sym_idx + 1].asnumpy() + + #if DEBUG: + # print 'im_size: ({}, {})'.format(im_info[0], im_info[1]) + # print 'scale: {}'.format(im_info[2]) + + #_height, _width = int(im_info[0] / stride), int(im_info[1] / stride) + height, width = bbox_deltas.shape[ + 2], bbox_deltas.shape[3] + + K = height * width + anchors_fpn = self._anchors_fpn['stride%s' % s] + anchors = anchors_plane(height, width, stride, + anchors_fpn) + #print((height, width), (_height, _width), anchors.shape, bbox_deltas.shape, scores.shape, file=sys.stderr) + anchors = anchors.reshape((K * A, 4)) + #print('num_anchors', self._num_anchors['stride%s'%s], file=sys.stderr) + #print('HW', (height, width), file=sys.stderr) + #print('anchors_fpn', anchors_fpn.shape, file=sys.stderr) + #print('anchors', anchors.shape, file=sys.stderr) + #print('bbox_deltas', bbox_deltas.shape, file=sys.stderr) + #print('scores', scores.shape, file=sys.stderr) + + #scores = self._clip_pad(scores, (height, width)) + scores = scores.transpose((0, 2, 3, 1)).reshape( + (-1, 1)) + mask_scores = mask_scores.transpose( + (0, 2, 3, 1)).reshape((-1, 1)) + + #print('pre', bbox_deltas.shape, height, width) + #bbox_deltas = self._clip_pad(bbox_deltas, (height, width)) + #print('after', bbox_deltas.shape, height, width) + bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)) + bbox_pred_len = bbox_deltas.shape[3] // A + #print(bbox_deltas.shape) + bbox_deltas = bbox_deltas.reshape((-1, bbox_pred_len)) + bbox_deltas[:, + 0::4] = bbox_deltas[:, 0:: + 4] * self.bbox_stds[0] + bbox_deltas[:, + 1::4] = bbox_deltas[:, 1:: + 4] * self.bbox_stds[1] + bbox_deltas[:, + 2::4] = bbox_deltas[:, 2:: + 4] * self.bbox_stds[2] + bbox_deltas[:, + 3::4] = bbox_deltas[:, 3:: + 4] * self.bbox_stds[3] + proposals = self.bbox_pred(anchors, bbox_deltas) + + proposals = clip_boxes(proposals, im_info[:2]) + + #if self.vote: + # if im_scale>1.0: + # keep = self._filter_boxes2(proposals, 160*im_scale, -1) + # else: + # keep = self._filter_boxes2(proposals, -1, 100*im_scale) + # if stride==4: + # keep = self._filter_boxes2(proposals, 12*im_scale, -1) + # proposals = proposals[keep, :] + # scores = scores[keep] + + #keep = self._filter_boxes(proposals, min_size_dict['stride%s'%s] * im_info[2]) + #proposals = proposals[keep, :] + #scores = scores[keep] + #print('333', proposals.shape) + if stride == 4 and self.decay4 < 1.0: + scores *= self.decay4 + + scores_ravel = scores.ravel() + #mask_scores_ravel = mask_scores.ravel() + #print('__shapes', proposals.shape, scores_ravel.shape) + #print('max score', np.max(scores_ravel)) + order = np.where(scores_ravel >= threshold)[0] + #_scores = scores_ravel[order] + #_order = _scores.argsort()[::-1] + #order = order[_order] + proposals = proposals[order, :] + scores = scores[order] + mask_scores = mask_scores[order] + if flip: + oldx1 = proposals[:, 0].copy() + oldx2 = proposals[:, 2].copy() + proposals[:, 0] = im.shape[1] - oldx2 - 1 + proposals[:, 2] = im.shape[1] - oldx1 - 1 + + proposals[:, 0:4] /= im_scale + + proposals_list.append(proposals) + scores_list.append(scores) + mask_scores_list.append(mask_scores) + + landmark_deltas = net_out[sym_idx + 2].asnumpy() + #landmark_deltas = self._clip_pad(landmark_deltas, (height, width)) + landmark_pred_len = landmark_deltas.shape[1] // A + landmark_deltas = landmark_deltas.transpose( + (0, 2, 3, 1)).reshape( + (-1, 5, landmark_pred_len // 5)) + landmark_deltas *= self.landmark_std + #print(landmark_deltas.shape, landmark_deltas) + landmarks = self.landmark_pred(anchors, + landmark_deltas) + landmarks = landmarks[order, :] + + if flip: + landmarks[:, :, + 0] = im.shape[1] - landmarks[:, :, 0] - 1 + #for a in range(5): + # oldx1 = landmarks[:, a].copy() + # landmarks[:,a] = im.shape[1] - oldx1 - 1 + order = [1, 0, 2, 4, 3] + flandmarks = landmarks.copy() + for idx, a in enumerate(order): + flandmarks[:, idx, :] = landmarks[:, a, :] + #flandmarks[:, idx*2] = landmarks[:,a*2] + #flandmarks[:, idx*2+1] = landmarks[:,a*2+1] + landmarks = flandmarks + landmarks[:, :, 0:2] /= im_scale + #landmarks /= im_scale + #landmarks = landmarks.reshape( (-1, landmark_pred_len) ) + landmarks_list.append(landmarks) + #proposals = np.hstack((proposals, landmarks)) + sym_idx += 4 + + if self.debug: + timeb = datetime.datetime.now() + diff = timeb - timea + print('B uses', diff.total_seconds(), 'seconds') + proposals = np.vstack(proposals_list) + landmarks = None + if proposals.shape[0] == 0: + landmarks = np.zeros((0, 5, 2)) + return np.zeros((0, 6)), landmarks + scores = np.vstack(scores_list) + mask_scores = np.vstack(mask_scores_list) + #print('shapes', proposals.shape, scores.shape) + scores_ravel = scores.ravel() + order = scores_ravel.argsort()[::-1] + #if config.TEST.SCORE_THRESH>0.0: + # _count = np.sum(scores_ravel>config.TEST.SCORE_THRESH) + # order = order[:_count] + proposals = proposals[order, :] + scores = scores[order] + mask_scores = mask_scores[order] + landmarks = np.vstack(landmarks_list) + landmarks = landmarks[order].astype(np.float32, copy=False) + + pre_det = np.hstack((proposals[:, 0:4], scores)).astype(np.float32, + copy=False) + keep = self.nms(pre_det) + det = np.hstack((pre_det, mask_scores)) + det = det[keep, :] + landmarks = landmarks[keep] + + if self.debug: + timeb = datetime.datetime.now() + diff = timeb - timea + print('C uses', diff.total_seconds(), 'seconds') + return det, landmarks + + def detect_center(self, img, threshold=0.5, scales=[1.0], do_flip=False): + det, landmarks = self.detect(img, threshold, scales, do_flip) + if det.shape[0] == 0: + return None, None + bindex = 0 + if det.shape[0] > 1: + img_size = np.asarray(img.shape)[0:2] + bounding_box_size = (det[:, 2] - det[:, 0]) * (det[:, 3] - + det[:, 1]) + img_center = img_size / 2 + offsets = np.vstack([(det[:, 0] + det[:, 2]) / 2 - img_center[1], + (det[:, 1] + det[:, 3]) / 2 - img_center[0]]) + offset_dist_squared = np.sum(np.power(offsets, 2.0), 0) + bindex = np.argmax(bounding_box_size - offset_dist_squared * + 2.0) # some extra weight on the centering + bbox = det[bindex, :] + landmark = landmarks[bindex, :, :] + return bbox, landmark + + @staticmethod + def check_large_pose(landmark, bbox): + assert landmark.shape == (5, 2) + assert len(bbox) == 4 + + def get_theta(base, x, y): + vx = x - base + vy = y - base + vx[1] *= -1 + vy[1] *= -1 + tx = np.arctan2(vx[1], vx[0]) + ty = np.arctan2(vy[1], vy[0]) + d = ty - tx + d = np.degrees(d) + #print(vx, tx, vy, ty, d) + #if d<-1.*math.pi: + # d+=2*math.pi + #elif d>math.pi: + # d-=2*math.pi + if d < -180.0: + d += 360. + elif d > 180.0: + d -= 360.0 + return d + + landmark = landmark.astype(np.float32) + + theta1 = get_theta(landmark[0], landmark[3], landmark[2]) + theta2 = get_theta(landmark[1], landmark[2], landmark[4]) + #print(va, vb, theta2) + theta3 = get_theta(landmark[0], landmark[2], landmark[1]) + theta4 = get_theta(landmark[1], landmark[0], landmark[2]) + theta5 = get_theta(landmark[3], landmark[4], landmark[2]) + theta6 = get_theta(landmark[4], landmark[2], landmark[3]) + theta7 = get_theta(landmark[3], landmark[2], landmark[0]) + theta8 = get_theta(landmark[4], landmark[1], landmark[2]) + #print(theta1, theta2, theta3, theta4, theta5, theta6, theta7, theta8) + left_score = 0.0 + right_score = 0.0 + up_score = 0.0 + down_score = 0.0 + if theta1 <= 0.0: + left_score = 10.0 + elif theta2 <= 0.0: + right_score = 10.0 + else: + left_score = theta2 / theta1 + right_score = theta1 / theta2 + if theta3 <= 10.0 or theta4 <= 10.0: + up_score = 10.0 + else: + up_score = max(theta1 / theta3, theta2 / theta4) + if theta5 <= 10.0 or theta6 <= 10.0: + down_score = 10.0 + else: + down_score = max(theta7 / theta5, theta8 / theta6) + mleft = (landmark[0][0] + landmark[3][0]) / 2 + mright = (landmark[1][0] + landmark[4][0]) / 2 + box_center = ((bbox[0] + bbox[2]) / 2, (bbox[1] + bbox[3]) / 2) + ret = 0 + if left_score >= 3.0: + ret = 1 + if ret == 0 and left_score >= 2.0: + if mright <= box_center[0]: + ret = 1 + if ret == 0 and right_score >= 3.0: + ret = 2 + if ret == 0 and right_score >= 2.0: + if mleft >= box_center[0]: + ret = 2 + if ret == 0 and up_score >= 2.0: + ret = 3 + if ret == 0 and down_score >= 5.0: + ret = 4 + return ret, left_score, right_score, up_score, down_score + + @staticmethod + def _filter_boxes(boxes, min_size): + """ Remove all boxes with any side smaller than min_size """ + ws = boxes[:, 2] - boxes[:, 0] + 1 + hs = boxes[:, 3] - boxes[:, 1] + 1 + keep = np.where((ws >= min_size) & (hs >= min_size))[0] + return keep + + @staticmethod + def _filter_boxes2(boxes, max_size, min_size): + """ Remove all boxes with any side smaller than min_size """ + ws = boxes[:, 2] - boxes[:, 0] + 1 + hs = boxes[:, 3] - boxes[:, 1] + 1 + if max_size > 0: + keep = np.where(np.minimum(ws, hs) < max_size)[0] + elif min_size > 0: + keep = np.where(np.maximum(ws, hs) > min_size)[0] + return keep + + @staticmethod + def _clip_pad(tensor, pad_shape): + """ + Clip boxes of the pad area. + :param tensor: [n, c, H, W] + :param pad_shape: [h, w] + :return: [n, c, h, w] + """ + H, W = tensor.shape[2:] + h, w = pad_shape + + if h < H or w < W: + tensor = tensor[:, :, :h, :w].copy() + + return tensor + + @staticmethod + def bbox_pred(boxes, box_deltas): + """ + Transform the set of class-agnostic boxes into class-specific boxes + by applying the predicted offsets (box_deltas) + :param boxes: !important [N 4] + :param box_deltas: [N, 4 * num_classes] + :return: [N 4 * num_classes] + """ + if boxes.shape[0] == 0: + return np.zeros((0, box_deltas.shape[1])) + + boxes = boxes.astype(np.float, copy=False) + widths = boxes[:, 2] - boxes[:, 0] + 1.0 + heights = boxes[:, 3] - boxes[:, 1] + 1.0 + ctr_x = boxes[:, 0] + 0.5 * (widths - 1.0) + ctr_y = boxes[:, 1] + 0.5 * (heights - 1.0) + + dx = box_deltas[:, 0:1] + dy = box_deltas[:, 1:2] + dw = box_deltas[:, 2:3] + dh = box_deltas[:, 3:4] + + pred_ctr_x = dx * widths[:, np.newaxis] + ctr_x[:, np.newaxis] + pred_ctr_y = dy * heights[:, np.newaxis] + ctr_y[:, np.newaxis] + pred_w = np.exp(dw) * widths[:, np.newaxis] + pred_h = np.exp(dh) * heights[:, np.newaxis] + + pred_boxes = np.zeros(box_deltas.shape) + # x1 + pred_boxes[:, 0:1] = pred_ctr_x - 0.5 * (pred_w - 1.0) + # y1 + pred_boxes[:, 1:2] = pred_ctr_y - 0.5 * (pred_h - 1.0) + # x2 + pred_boxes[:, 2:3] = pred_ctr_x + 0.5 * (pred_w - 1.0) + # y2 + pred_boxes[:, 3:4] = pred_ctr_y + 0.5 * (pred_h - 1.0) + + if box_deltas.shape[1] > 4: + pred_boxes[:, 4:] = box_deltas[:, 4:] + + return pred_boxes + + @staticmethod + def landmark_pred(boxes, landmark_deltas): + if boxes.shape[0] == 0: + return np.zeros((0, landmark_deltas.shape[1])) + boxes = boxes.astype(np.float, copy=False) + widths = boxes[:, 2] - boxes[:, 0] + 1.0 + heights = boxes[:, 3] - boxes[:, 1] + 1.0 + ctr_x = boxes[:, 0] + 0.5 * (widths - 1.0) + ctr_y = boxes[:, 1] + 0.5 * (heights - 1.0) + pred = landmark_deltas.copy() + for i in range(5): + pred[:, i, 0] = landmark_deltas[:, i, 0] * widths + ctr_x + pred[:, i, 1] = landmark_deltas[:, i, 1] * heights + ctr_y + return pred + #preds = [] + #for i in range(landmark_deltas.shape[1]): + # if i%2==0: + # pred = (landmark_deltas[:,i]*widths + ctr_x) + # else: + # pred = (landmark_deltas[:,i]*heights + ctr_y) + # preds.append(pred) + #preds = np.vstack(preds).transpose() + #return preds + + def vote(self, det): + #order = det[:, 4].ravel().argsort()[::-1] + #det = det[order, :] + if det.shape[0] == 0: + return np.zeros((0, 5)) + #dets = np.array([[10, 10, 20, 20, 0.002]]) + #det = np.empty(shape=[0, 5]) + dets = None + while det.shape[0] > 0: + if dets is not None and dets.shape[0] >= 750: + break + # IOU + area = (det[:, 2] - det[:, 0] + 1) * (det[:, 3] - det[:, 1] + 1) + xx1 = np.maximum(det[0, 0], det[:, 0]) + yy1 = np.maximum(det[0, 1], det[:, 1]) + xx2 = np.minimum(det[0, 2], det[:, 2]) + yy2 = np.minimum(det[0, 3], det[:, 3]) + w = np.maximum(0.0, xx2 - xx1 + 1) + h = np.maximum(0.0, yy2 - yy1 + 1) + inter = w * h + o = inter / (area[0] + area[:] - inter) + + # nms + merge_index = np.where(o >= self.nms_threshold)[0] + det_accu = det[merge_index, :] + det = np.delete(det, merge_index, 0) + if merge_index.shape[0] <= 1: + if det.shape[0] == 0: + try: + dets = np.row_stack((dets, det_accu)) + except: + dets = det_accu + continue + det_accu[:, + 0:4] = det_accu[:, 0:4] * np.tile(det_accu[:, -1:], + (1, 4)) + max_score = np.max(det_accu[:, 4]) + det_accu_sum = np.zeros((1, 5)) + det_accu_sum[:, 0:4] = np.sum(det_accu[:, 0:4], axis=0) / np.sum( + det_accu[:, -1:]) + det_accu_sum[:, 4] = max_score + if dets is None: + dets = det_accu_sum + else: + dets = np.row_stack((dets, det_accu_sum)) + dets = dets[0:750, :] + return dets diff --git a/detection/RetinaFaceAntiCov/test.py b/detection/RetinaFaceAntiCov/test.py new file mode 100644 index 0000000..f7c282b --- /dev/null +++ b/detection/RetinaFaceAntiCov/test.py @@ -0,0 +1,66 @@ +import cv2 +import sys +import numpy as np +import datetime +import os +import glob +from retinaface_cov import RetinaFaceCoV + +thresh = 0.8 +mask_thresh = 0.2 +scales = [640, 1080] + +count = 1 + +gpuid = 0 +#detector = RetinaFaceCoV('./model/mnet_cov1', 0, gpuid, 'net3') +detector = RetinaFaceCoV('./model/mnet_cov2', 0, gpuid, 'net3l') + +img = cv2.imread('n1.jpg') +print(img.shape) +im_shape = img.shape +target_size = scales[0] +max_size = scales[1] +im_size_min = np.min(im_shape[0:2]) +im_size_max = np.max(im_shape[0:2]) +#im_scale = 1.0 +#if im_size_min>target_size or im_size_max>max_size: +im_scale = float(target_size) / float(im_size_min) +# prevent bigger axis from being more than max_size: +if np.round(im_scale * im_size_max) > max_size: + im_scale = float(max_size) / float(im_size_max) + +print('im_scale', im_scale) + +scales = [im_scale] +flip = False + +for c in range(count): + faces, landmarks = detector.detect(img, + thresh, + scales=scales, + do_flip=flip) + +if faces is not None: + print('find', faces.shape[0], 'faces') + for i in range(faces.shape[0]): + #print('score', faces[i][4]) + face = faces[i] + box = face[0:4].astype(np.int) + mask = face[5] + print(i, box, mask) + #color = (255,0,0) + if mask >= mask_thresh: + color = (0, 0, 255) + else: + color = (0, 255, 0) + cv2.rectangle(img, (box[0], box[1]), (box[2], box[3]), color, 2) + landmark5 = landmarks[i].astype(np.int) + #print(landmark.shape) + for l in range(landmark5.shape[0]): + color = (255, 0, 0) + cv2.circle(img, (landmark5[l][0], landmark5[l][1]), 1, color, 2) + + filename = './cov_test.jpg' + print('writing', filename) + cv2.imwrite(filename, img) diff --git a/Evaluation/IJB/IJBB_Evaluation_MS1MV2.ipynb b/evaluation/IJB/IJBB_Evaluation_MS1MV2.ipynb similarity index 100% rename from Evaluation/IJB/IJBB_Evaluation_MS1MV2.ipynb rename to evaluation/IJB/IJBB_Evaluation_MS1MV2.ipynb diff --git a/Evaluation/IJB/IJBB_Evaluation_VGG2.ipynb b/evaluation/IJB/IJBB_Evaluation_VGG2.ipynb similarity index 100% rename from Evaluation/IJB/IJBB_Evaluation_VGG2.ipynb rename to evaluation/IJB/IJBB_Evaluation_VGG2.ipynb diff --git a/Evaluation/IJB/IJBC_Evaluation_MS1MV2.ipynb b/evaluation/IJB/IJBC_Evaluation_MS1MV2.ipynb similarity index 100% rename from Evaluation/IJB/IJBC_Evaluation_MS1MV2.ipynb rename to evaluation/IJB/IJBC_Evaluation_MS1MV2.ipynb diff --git a/Evaluation/IJB/IJBC_Evaluation_VGG2.ipynb b/evaluation/IJB/IJBC_Evaluation_VGG2.ipynb similarity index 100% rename from Evaluation/IJB/IJBC_Evaluation_VGG2.ipynb rename to evaluation/IJB/IJBC_Evaluation_VGG2.ipynb diff --git a/Evaluation/IJB/IJB_11.py b/evaluation/IJB/IJB_11.py similarity index 62% rename from Evaluation/IJB/IJB_11.py rename to evaluation/IJB/IJB_11.py index ce3f9fe..53b3326 100644 --- a/Evaluation/IJB/IJB_11.py +++ b/evaluation/IJB/IJB_11.py @@ -22,8 +22,8 @@ from menpo.visualize import print_progress from menpo.visualize.viewmatplotlib import sample_colours_from_colourmap from prettytable import PrettyTable from pathlib import Path -import warnings -warnings.filterwarnings("ignore") +import warnings +warnings.filterwarnings("ignore") parser = argparse.ArgumentParser(description='do ijb test') # general @@ -32,26 +32,30 @@ parser.add_argument('--model-epoch', default=1, type=int, help='') parser.add_argument('--gpu', default=7, type=int, help='gpu id') parser.add_argument('--batch-size', default=32, type=int, help='') parser.add_argument('--job', default='insightface', type=str, help='job name') -parser.add_argument('--target', default='IJBC', type=str, help='target, set to IJBC or IJBB') +parser.add_argument('--target', + default='IJBC', + type=str, + help='target, set to IJBC or IJBB') args = parser.parse_args() target = args.target model_path = args.model_prefix gpu_id = args.gpu epoch = args.model_epoch -use_norm_score = True # if Ture, TestMode(N1) -use_detector_score = True # if Ture, TestMode(D1) -use_flip_test = True # if Ture, TestMode(F1) +use_norm_score = True # if Ture, TestMode(N1) +use_detector_score = True # if Ture, TestMode(D1) +use_flip_test = True # if Ture, TestMode(F1) job = args.job def read_template_media_list(path): #ijb_meta = np.loadtxt(path, dtype=str) ijb_meta = pd.read_csv(path, sep=' ', header=None).values - templates = ijb_meta[:,1].astype(np.int) - medias = ijb_meta[:,2].astype(np.int) + templates = ijb_meta[:, 1].astype(np.int) + medias = ijb_meta[:, 2].astype(np.int) return templates, medias + # In[ ]: @@ -60,11 +64,12 @@ def read_template_pair_list(path): pairs = pd.read_csv(path, sep=' ', header=None).values #print(pairs.shape) #print(pairs[:, 0].astype(np.int)) - t1 = pairs[:,0].astype(np.int) - t2 = pairs[:,1].astype(np.int) - label = pairs[:,2].astype(np.int) + t1 = pairs[:, 0].astype(np.int) + t2 = pairs[:, 1].astype(np.int) + label = pairs[:, 2].astype(np.int) return t1, t2, label + # In[ ]: @@ -73,6 +78,7 @@ def read_image_feature(path): img_feats = pickle.load(fid) return img_feats + # In[ ]: @@ -84,14 +90,15 @@ def get_image_feature(img_path, img_list_path, model_path, epoch, gpu_id): faceness_scores = [] img_feats = [] for img_index, each_line in enumerate(files): - if img_index%500==0: - print('processing', img_index) + if img_index % 500 == 0: + print('processing', img_index) name_lmk_score = each_line.strip().split(' ') img_name = os.path.join(img_path, name_lmk_score[0]) img = cv2.imread(img_name) - lmk = np.array([float(x) for x in name_lmk_score[1:-1]], dtype=np.float32) - lmk = lmk.reshape( (5,2) ) - img_feats.append(embedding.get(img,lmk)) + lmk = np.array([float(x) for x in name_lmk_score[1:-1]], + dtype=np.float32) + lmk = lmk.reshape((5, 2)) + img_feats.append(embedding.get(img, lmk)) faceness_scores.append(name_lmk_score[-1]) img_feats = np.array(img_feats).astype(np.float32) faceness_scores = np.array(faceness_scores).astype(np.float32) @@ -100,56 +107,67 @@ def get_image_feature(img_path, img_list_path, model_path, epoch, gpu_id): #faceness_scores = np.ones( (len(files), ), dtype=np.float32 ) return img_feats, faceness_scores + # In[ ]: -def image2template_feature(img_feats = None, templates = None, medias = None): +def image2template_feature(img_feats=None, templates=None, medias=None): # ========================================================== # 1. face image feature l2 normalization. img_feats:[number_image x feats_dim] # 2. compute media feature. # 3. compute template feature. - # ========================================================== + # ========================================================== unique_templates = np.unique(templates) template_feats = np.zeros((len(unique_templates), img_feats.shape[1])) for count_template, uqt in enumerate(unique_templates): - (ind_t,) = np.where(templates == uqt) + (ind_t, ) = np.where(templates == uqt) face_norm_feats = img_feats[ind_t] face_medias = medias[ind_t] - unique_medias, unique_media_counts = np.unique(face_medias, return_counts=True) + unique_medias, unique_media_counts = np.unique(face_medias, + return_counts=True) media_norm_feats = [] - for u,ct in zip(unique_medias, unique_media_counts): - (ind_m,) = np.where(face_medias == u) + for u, ct in zip(unique_medias, unique_media_counts): + (ind_m, ) = np.where(face_medias == u) if ct == 1: media_norm_feats += [face_norm_feats[ind_m]] - else: # image features from the same video will be aggregated into one feature - media_norm_feats += [np.mean(face_norm_feats[ind_m], axis=0, keepdims=True)] + else: # image features from the same video will be aggregated into one feature + media_norm_feats += [ + np.mean(face_norm_feats[ind_m], axis=0, keepdims=True) + ] media_norm_feats = np.array(media_norm_feats) # media_norm_feats = media_norm_feats / np.sqrt(np.sum(media_norm_feats ** 2, -1, keepdims=True)) template_feats[count_template] = np.sum(media_norm_feats, axis=0) - if count_template % 2000 == 0: - print('Finish Calculating {} template features.'.format(count_template)) + if count_template % 2000 == 0: + print('Finish Calculating {} template features.'.format( + count_template)) #template_norm_feats = template_feats / np.sqrt(np.sum(template_feats ** 2, -1, keepdims=True)) template_norm_feats = sklearn.preprocessing.normalize(template_feats) #print(template_norm_feats.shape) return template_norm_feats, unique_templates + # In[ ]: -def verification(template_norm_feats = None, unique_templates = None, p1 = None, p2 = None): +def verification(template_norm_feats=None, + unique_templates=None, + p1=None, + p2=None): # ========================================================== # Compute set-to-set Similarity Score. # ========================================================== - template2id = np.zeros((max(unique_templates)+1,1),dtype=int) + template2id = np.zeros((max(unique_templates) + 1, 1), dtype=int) for count_template, uqt in enumerate(unique_templates): template2id[uqt] = count_template - - score = np.zeros((len(p1),)) # save cosine distance between pairs + + score = np.zeros((len(p1), )) # save cosine distance between pairs total_pairs = np.array(range(len(p1))) - batchsize = 100000 # small batchsize instead of all pairs in one batch due to the memory limiation - sublists = [total_pairs[i:i + batchsize] for i in range(0, len(p1), batchsize)] + batchsize = 100000 # small batchsize instead of all pairs in one batch due to the memory limiation + sublists = [ + total_pairs[i:i + batchsize] for i in range(0, len(p1), batchsize) + ] total_sublists = len(sublists) for c, s in enumerate(sublists): feat1 = template_norm_feats[template2id[p1[s]]] @@ -160,24 +178,30 @@ def verification(template_norm_feats = None, unique_templates = None, p1 = None, print('Finish {}/{} pairs.'.format(c, total_sublists)) return score + # In[ ]: -def verification2(template_norm_feats = None, unique_templates = None, p1 = None, p2 = None): - template2id = np.zeros((max(unique_templates) + 1, 1), dtype=int) - for count_template, uqt in enumerate(unique_templates): - template2id[uqt] = count_template - score = np.zeros((len(p1),)) # save cosine distance between pairs - total_pairs = np.array(range(len(p1))) - batchsize = 100000 # small batchsize instead of all pairs in one batch due to the memory limiation - sublists = [total_pairs[i:i + batchsize] for i in range(0, len(p1), batchsize)] - total_sublists = len(sublists) - for c, s in enumerate(sublists): - feat1 = template_norm_feats[template2id[p1[s]]] - feat2 = template_norm_feats[template2id[p2[s]]] - similarity_score = np.sum(feat1 * feat2, -1) - score[s] = similarity_score.flatten() - if c % 10 == 0: - print('Finish {}/{} pairs.'.format(c, total_sublists)) - return score +def verification2(template_norm_feats=None, + unique_templates=None, + p1=None, + p2=None): + template2id = np.zeros((max(unique_templates) + 1, 1), dtype=int) + for count_template, uqt in enumerate(unique_templates): + template2id[uqt] = count_template + score = np.zeros((len(p1), )) # save cosine distance between pairs + total_pairs = np.array(range(len(p1))) + batchsize = 100000 # small batchsize instead of all pairs in one batch due to the memory limiation + sublists = [ + total_pairs[i:i + batchsize] for i in range(0, len(p1), batchsize) + ] + total_sublists = len(sublists) + for c, s in enumerate(sublists): + feat1 = template_norm_feats[template2id[p1[s]]] + feat2 = template_norm_feats[template2id[p2[s]]] + similarity_score = np.sum(feat1 * feat2, -1) + score[s] = similarity_score.flatten() + if c % 10 == 0: + print('Finish {}/{} pairs.'.format(c, total_sublists)) + return score def read_score(path): @@ -185,26 +209,27 @@ def read_score(path): img_feats = pickle.load(fid) return img_feats + # # Step1: Load Meta Data # In[ ]: -assert target=='IJBC' or target=='IJBB' +assert target == 'IJBC' or target == 'IJBB' # ============================================================= # load image and template relationships for template feature embedding -# tid --> template id, mid --> media id +# tid --> template id, mid --> media id # format: # image_name tid mid # ============================================================= start = timeit.default_timer() -templates, medias = read_template_media_list(os.path.join('%s/meta'%target, '%s_face_tid_mid.txt'%target.lower())) +templates, medias = read_template_media_list( + os.path.join('%s/meta' % target, '%s_face_tid_mid.txt' % target.lower())) stop = timeit.default_timer() print('Time: %.2f s. ' % (stop - start)) # In[ ]: - # ============================================================= # load template pairs for template-to-template verification # tid : template id, label : 1/0 @@ -212,7 +237,9 @@ print('Time: %.2f s. ' % (stop - start)) # tid_1 tid_2 label # ============================================================= start = timeit.default_timer() -p1, p2, label = read_template_pair_list(os.path.join('%s/meta'%target, '%s_template_pair_label.txt'%target.lower())) +p1, p2, label = read_template_pair_list( + os.path.join('%s/meta' % target, + '%s_template_pair_label.txt' % target.lower())) stop = timeit.default_timer() print('Time: %.2f s. ' % (stop - start)) @@ -220,59 +247,61 @@ print('Time: %.2f s. ' % (stop - start)) # In[ ]: - # ============================================================= -# load image features +# load image features # format: # img_feats: [image_num x feats_dim] (227630, 512) # ============================================================= start = timeit.default_timer() img_path = './%s/loose_crop' % target img_list_path = './%s/meta/%s_name_5pts_score.txt' % (target, target.lower()) -img_feats, faceness_scores = get_image_feature(img_path, img_list_path, model_path, epoch, gpu_id) +img_feats, faceness_scores = get_image_feature(img_path, img_list_path, + model_path, epoch, gpu_id) stop = timeit.default_timer() print('Time: %.2f s. ' % (stop - start)) -print('Feature Shape: ({} , {}) .'.format(img_feats.shape[0], img_feats.shape[1])) +print('Feature Shape: ({} , {}) .'.format(img_feats.shape[0], + img_feats.shape[1])) # # Step3: Get Template Features # In[ ]: - # ============================================================= # compute template features from image features. # ============================================================= start = timeit.default_timer() -# ========================================================== +# ========================================================== # Norm feature before aggregation into template feature? # Feature norm from embedding network and faceness score are able to decrease weights for noise samples (not face). -# ========================================================== +# ========================================================== # 1. FaceScore （Feature Norm） # 2. FaceScore （Detector） - if use_flip_test: # concat --- F1 - #img_input_feats = img_feats + #img_input_feats = img_feats # add --- F2 - img_input_feats = img_feats[:,0:img_feats.shape[1]//2] + img_feats[:,img_feats.shape[1]//2:] + img_input_feats = img_feats[:, 0:img_feats.shape[1] // + 2] + img_feats[:, img_feats.shape[1] // 2:] else: - img_input_feats = img_feats[:,0:img_feats.shape[1]//2] - + img_input_feats = img_feats[:, 0:img_feats.shape[1] // 2] + if use_norm_score: img_input_feats = img_input_feats else: # normalise features to remove norm information - img_input_feats = img_input_feats / np.sqrt(np.sum(img_input_feats ** 2, -1, keepdims=True)) - + img_input_feats = img_input_feats / np.sqrt( + np.sum(img_input_feats**2, -1, keepdims=True)) + if use_detector_score: print(img_input_feats.shape, faceness_scores.shape) #img_input_feats = img_input_feats * np.matlib.repmat(faceness_scores[:,np.newaxis], 1, img_input_feats.shape[1]) - img_input_feats = img_input_feats * faceness_scores[:,np.newaxis] + img_input_feats = img_input_feats * faceness_scores[:, np.newaxis] else: img_input_feats = img_input_feats -template_norm_feats, unique_templates = image2template_feature(img_input_feats, templates, medias) +template_norm_feats, unique_templates = image2template_feature( + img_input_feats, templates, medias) stop = timeit.default_timer() print('Time: %.2f s. ' % (stop - start)) @@ -280,7 +309,6 @@ print('Time: %.2f s. ' % (stop - start)) # In[ ]: - # ============================================================= # compute verification scores between template pairs. # ============================================================= @@ -296,52 +324,57 @@ save_path = './%s_result' % target if not os.path.exists(save_path): os.makedirs(save_path) -score_save_file = os.path.join(save_path, "%s.npy"%job) +score_save_file = os.path.join(save_path, "%s.npy" % job) np.save(score_save_file, score) # # Step 5: Get ROC Curves and TPR@FPR Table # In[ ]: - files = [score_save_file] methods = [] scores = [] for file in files: methods.append(Path(file).stem) - scores.append(np.load(file)) + scores.append(np.load(file)) methods = np.array(methods) -scores = dict(zip(methods,scores)) -colours = dict(zip(methods, sample_colours_from_colourmap(methods.shape[0], 'Set2'))) +scores = dict(zip(methods, scores)) +colours = dict( + zip(methods, sample_colours_from_colourmap(methods.shape[0], 'Set2'))) #x_labels = [1/(10**x) for x in np.linspace(6, 0, 6)] -x_labels = [10**-6, 10**-5, 10**-4,10**-3, 10**-2, 10**-1] +x_labels = [10**-6, 10**-5, 10**-4, 10**-3, 10**-2, 10**-1] tpr_fpr_table = PrettyTable(['Methods'] + [str(x) for x in x_labels]) fig = plt.figure() for method in methods: fpr, tpr, _ = roc_curve(label, scores[method]) roc_auc = auc(fpr, tpr) fpr = np.flipud(fpr) - tpr = np.flipud(tpr) # select largest tpr at same fpr - plt.plot(fpr, tpr, color=colours[method], lw=1, label=('[%s (AUC = %0.4f %%)]' % (method.split('-')[-1], roc_auc*100))) + tpr = np.flipud(tpr) # select largest tpr at same fpr + plt.plot(fpr, + tpr, + color=colours[method], + lw=1, + label=('[%s (AUC = %0.4f %%)]' % + (method.split('-')[-1], roc_auc * 100))) tpr_fpr_row = [] - tpr_fpr_row.append("%s-%s"%(method, target)) + tpr_fpr_row.append("%s-%s" % (method, target)) for fpr_iter in np.arange(len(x_labels)): - _, min_index = min(list(zip(abs(fpr-x_labels[fpr_iter]), range(len(fpr))))) + _, min_index = min( + list(zip(abs(fpr - x_labels[fpr_iter]), range(len(fpr))))) #tpr_fpr_row.append('%.4f' % tpr[min_index]) - tpr_fpr_row.append('%.2f' % (tpr[min_index]*100)) + tpr_fpr_row.append('%.2f' % (tpr[min_index] * 100)) tpr_fpr_table.add_row(tpr_fpr_row) plt.xlim([10**-6, 0.1]) plt.ylim([0.3, 1.0]) plt.grid(linestyle='--', linewidth=1) -plt.xticks(x_labels) -plt.yticks(np.linspace(0.3, 1.0, 8, endpoint=True)) +plt.xticks(x_labels) +plt.yticks(np.linspace(0.3, 1.0, 8, endpoint=True)) plt.xscale('log') plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.title('ROC on IJB') plt.legend(loc="lower right") #plt.show() -fig.savefig(os.path.join(save_path, '%s.pdf'%job)) +fig.savefig(os.path.join(save_path, '%s.pdf' % job)) print(tpr_fpr_table) - diff --git a/evaluation/IJB/IJB_1N.py b/evaluation/IJB/IJB_1N.py new file mode 100644 index 0000000..85ea405 --- /dev/null +++ b/evaluation/IJB/IJB_1N.py @@ -0,0 +1,366 @@ +#!/usr/bin/env python +# coding: utf-8 +import os +import numpy as np +import timeit +import sklearn +import cv2 +import sys +import argparse +import glob +import numpy.matlib +import heapq +import math +from datetime import datetime as dt + +from sklearn import preprocessing +sys.path.append('./recognition') +from embedding import Embedding +from menpo.visualize import print_progress +from menpo.visualize.viewmatplotlib import sample_colours_from_colourmap + + +def read_template_subject_id_list(path): + ijb_meta = np.loadtxt(path, dtype=str, skiprows=1, delimiter=',') + templates = ijb_meta[:, 0].astype(np.int) + subject_ids = ijb_meta[:, 1].astype(np.int) + return templates, subject_ids + + +def read_template_media_list(path): + ijb_meta = np.loadtxt(path, dtype=str) + templates = ijb_meta[:, 1].astype(np.int) + medias = ijb_meta[:, 2].astype(np.int) + return templates, medias + + +def read_template_pair_list(path): + pairs = np.loadtxt(path, dtype=str) + t1 = pairs[:, 0].astype(np.int) + t2 = pairs[:, 1].astype(np.int) + label = pairs[:, 2].astype(np.int) + return t1, t2, label + + +#def get_image_feature(feature_path, faceness_path): +# img_feats = np.loadtxt(feature_path) +# faceness_scores = np.loadtxt(faceness_path) +# return img_feats, faceness_scores +def get_image_feature(img_path, img_list_path, model_path, epoch, gpu_id): + img_list = open(img_list_path) + embedding = Embedding(model_path, epoch, gpu_id) + files = img_list.readlines() + print('files:', len(files)) + faceness_scores = [] + img_feats = [] + for img_index, each_line in enumerate(files): + if img_index % 500 == 0: + print('processing', img_index) + name_lmk_score = each_line.strip().split(' ') + img_name = os.path.join(img_path, name_lmk_score[0]) + img = cv2.imread(img_name) + lmk = np.array([float(x) for x in name_lmk_score[1:-1]], + dtype=np.float32) + lmk = lmk.reshape((5, 2)) + img_feats.append(embedding.get(img, lmk)) + faceness_scores.append(name_lmk_score[-1]) + img_feats = np.array(img_feats).astype(np.float32) + faceness_scores = np.array(faceness_scores).astype(np.float32) + + #img_feats = np.ones( (len(files), 1024), dtype=np.float32) * 0.01 + #faceness_scores = np.ones( (len(files), ), dtype=np.float32 ) + return img_feats, faceness_scores + + +def image2template_feature(img_feats=None, + templates=None, + medias=None, + choose_templates=None, + choose_ids=None): + # ========================================================== + # 1. face image feature l2 normalization. img_feats:[number_image x feats_dim] + # 2. compute media feature. + # 3. compute template feature. + # ========================================================== + unique_templates, indices = np.unique(choose_templates, return_index=True) + unique_subjectids = choose_ids[indices] + template_feats = np.zeros((len(unique_templates), img_feats.shape[1])) + + for count_template, uqt in enumerate(unique_templates): + (ind_t, ) = np.where(templates == uqt) + face_norm_feats = img_feats[ind_t] + face_medias = medias[ind_t] + unique_medias, unique_media_counts = np.unique(face_medias, + return_counts=True) + media_norm_feats = [] + for u, ct in zip(unique_medias, unique_media_counts): + (ind_m, ) = np.where(face_medias == u) + if ct == 1: + media_norm_feats += [face_norm_feats[ind_m]] + else: # image features from the same video will be aggregated into one feature + media_norm_feats += [ + np.mean(face_norm_feats[ind_m], 0, keepdims=True) + ] + media_norm_feats = np.array(media_norm_feats) + # media_norm_feats = media_norm_feats / np.sqrt(np.sum(media_norm_feats ** 2, -1, keepdims=True)) + template_feats[count_template] = np.sum(media_norm_feats, 0) + if count_template % 2000 == 0: + print('Finish Calculating {} template features.'.format( + count_template)) + template_norm_feats = template_feats / np.sqrt( + np.sum(template_feats**2, -1, keepdims=True)) + return template_norm_feats, unique_templates, unique_subjectids + + +def verification(template_norm_feats=None, + unique_templates=None, + p1=None, + p2=None): + # ========================================================== + # Compute set-to-set Similarity Score. + # ========================================================== + template2id = np.zeros((max(unique_templates) + 1, 1), dtype=int) + for count_template, uqt in enumerate(unique_templates): + template2id[uqt] = count_template + + score = np.zeros((len(p1), )) # save cosine distance between pairs + + total_pairs = np.array(range(len(p1))) + batchsize = 100000 # small batchsize instead of all pairs in one batch due to the memory limiation + sublists = [ + total_pairs[i:i + batchsize] for i in range(0, len(p1), batchsize) + ] + total_sublists = len(sublists) + for c, s in enumerate(sublists): + feat1 = template_norm_feats[template2id[p1[s]]] + feat2 = template_norm_feats[template2id[p2[s]]] + similarity_score = np.sum(feat1 * feat2, -1) + score[s] = similarity_score.flatten() + if c % 10 == 0: + print('Finish {}/{} pairs.'.format(c, total_sublists)) + return score + + +def read_score(path): + with open(path, 'rb') as fid: + img_feats = cPickle.load(fid) + return img_feats + + +def evaluation(query_feats, gallery_feats, mask): + Fars = [0.01, 0.1] + print(query_feats.shape) + print(gallery_feats.shape) + + query_num = query_feats.shape[0] + gallery_num = gallery_feats.shape[0] + + similarity = np.dot(query_feats, gallery_feats.T) + print('similarity shape', similarity.shape) + top_inds = np.argsort(-similarity) + print(top_inds.shape) + + # calculate top1 + correct_num = 0 + for i in range(query_num): + j = top_inds[i, 0] + if j == mask[i]: + correct_num += 1 + print("top1 = {}".format(correct_num / query_num)) + # calculate top5 + correct_num = 0 + for i in range(query_num): + j = top_inds[i, 0:5] + if mask[i] in j: + correct_num += 1 + print("top5 = {}".format(correct_num / query_num)) + # calculate 10 + correct_num = 0 + for i in range(query_num): + j = top_inds[i, 0:10] + if mask[i] in j: + correct_num += 1 + print("top10 = {}".format(correct_num / query_num)) + + neg_pair_num = query_num * gallery_num - query_num + print(neg_pair_num) + required_topk = [math.ceil(query_num * x) for x in Fars] + top_sims = similarity + # calculate fars and tprs + pos_sims = [] + for i in range(query_num): + gt = mask[i] + pos_sims.append(top_sims[i, gt]) + top_sims[i, gt] = -2.0 + + pos_sims = np.array(pos_sims) + print(pos_sims.shape) + neg_sims = top_sims[np.where(top_sims > -2.0)] + print("neg_sims num = {}".format(len(neg_sims))) + neg_sims = heapq.nlargest(max(required_topk), neg_sims) # heap sort + print("after sorting , neg_sims num = {}".format(len(neg_sims))) + for far, pos in zip(Fars, required_topk): + th = neg_sims[pos - 1] + recall = np.sum(pos_sims > th) / query_num + print("far = {:.10f} pr = {:.10f} th = {:.10f}".format( + far, recall, th)) + + +def gen_mask(query_ids, reg_ids): + mask = [] + for query_id in query_ids: + pos = [i for i, x in enumerate(reg_ids) if query_id == x] + if len(pos) != 1: + raise RuntimeError( + "RegIdsError with id = {}， duplicate = {} ".format( + query_id, len(pos))) + mask.append(pos[0]) + return mask + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='do ijb 1n test') + # general + parser.add_argument('--model-prefix', + default='', + help='path to load model.') + parser.add_argument('--model-epoch', default=1, type=int, help='') + parser.add_argument('--gpu', default=7, type=int, help='gpu id') + parser.add_argument('--batch-size', default=32, type=int, help='') + parser.add_argument('--job', + default='insightface', + type=str, + help='job name') + parser.add_argument('--target', + default='IJBC', + type=str, + help='target, set to IJBC or IJBB') + args = parser.parse_args() + target = args.target + model_path = args.model_prefix + gpu_id = args.gpu + epoch = args.model_epoch + meta_dir = "%s/meta" % args.target #meta root dir + if target == 'IJBC': + gallery_s1_record = "%s_1N_gallery_G1.csv" % (args.target.lower()) + gallery_s2_record = "%s_1N_gallery_G2.csv" % (args.target.lower()) + else: + gallery_s1_record = "%s_1N_gallery_S1.csv" % (args.target.lower()) + gallery_s2_record = "%s_1N_gallery_S2.csv" % (args.target.lower()) + gallery_s1_templates, gallery_s1_subject_ids = read_template_subject_id_list( + os.path.join(meta_dir, gallery_s1_record)) + print(gallery_s1_templates.shape, gallery_s1_subject_ids.shape) + + gallery_s2_templates, gallery_s2_subject_ids = read_template_subject_id_list( + os.path.join(meta_dir, gallery_s2_record)) + print(gallery_s2_templates.shape, gallery_s2_templates.shape) + + gallery_templates = np.concatenate( + [gallery_s1_templates, gallery_s2_templates]) + gallery_subject_ids = np.concatenate( + [gallery_s1_subject_ids, gallery_s2_subject_ids]) + print(gallery_templates.shape, gallery_subject_ids.shape) + + media_record = "%s_face_tid_mid.txt" % args.target.lower() + total_templates, total_medias = read_template_media_list( + os.path.join(meta_dir, media_record)) + print("total_templates", total_templates.shape, total_medias.shape) + #load image features + start = timeit.default_timer() + feature_path = '' #feature path + face_path = '' #face path + img_path = './%s/loose_crop' % target + img_list_path = './%s/meta/%s_name_5pts_score.txt' % (target, + target.lower()) + #img_feats, faceness_scores = get_image_feature(feature_path, face_path) + img_feats, faceness_scores = get_image_feature(img_path, img_list_path, + model_path, epoch, gpu_id) + print('img_feats', img_feats.shape) + print('faceness_scores', faceness_scores.shape) + stop = timeit.default_timer() + print('Time: %.2f s. ' % (stop - start)) + print('Feature Shape: ({} , {}) .'.format(img_feats.shape[0], + img_feats.shape[1])) + + # compute template features from image features. + start = timeit.default_timer() + # ========================================================== + # Norm feature before aggregation into template feature? + # Feature norm from embedding network and faceness score are able to decrease weights for noise samples (not face). + # ========================================================== + use_norm_score = True # if True, TestMode(N1) + use_detector_score = True # if True, TestMode(D1) + use_flip_test = True # if True, TestMode(F1) + + if use_flip_test: + # concat --- F1 + #img_input_feats = img_feats + # add --- F2 + img_input_feats = img_feats[:, 0:int( + img_feats.shape[1] / 2)] + img_feats[:, + int(img_feats.shape[1] / 2):] + else: + img_input_feats = img_feats[:, 0:int(img_feats.shape[1] / 2)] + + if use_norm_score: + img_input_feats = img_input_feats + else: + # normalise features to remove norm information + img_input_feats = img_input_feats / np.sqrt( + np.sum(img_input_feats**2, -1, keepdims=True)) + + if use_detector_score: + img_input_feats = img_input_feats * np.matlib.repmat( + faceness_scores[:, np.newaxis], 1, img_input_feats.shape[1]) + else: + img_input_feats = img_input_feats + print("input features shape", img_input_feats.shape) + + #load gallery feature + gallery_templates_feature, gallery_unique_templates, gallery_unique_subject_ids = image2template_feature( + img_input_feats, total_templates, total_medias, gallery_templates, + gallery_subject_ids) + stop = timeit.default_timer() + print('Time: %.2f s. ' % (stop - start)) + print("gallery_templates_feature", gallery_templates_feature.shape) + print("gallery_unique_subject_ids", gallery_unique_subject_ids.shape) + #np.savetxt("gallery_templates_feature.txt", gallery_templates_feature) + #np.savetxt("gallery_unique_subject_ids.txt", gallery_unique_subject_ids) + + #load prope feature + probe_mixed_record = "%s_1N_probe_mixed.csv" % target.lower() + probe_mixed_templates, probe_mixed_subject_ids = read_template_subject_id_list( + os.path.join(meta_dir, probe_mixed_record)) + print(probe_mixed_templates.shape, probe_mixed_subject_ids.shape) + probe_mixed_templates_feature, probe_mixed_unique_templates, probe_mixed_unique_subject_ids = image2template_feature( + img_input_feats, total_templates, total_medias, probe_mixed_templates, + probe_mixed_subject_ids) + print("probe_mixed_templates_feature", probe_mixed_templates_feature.shape) + print("probe_mixed_unique_subject_ids", + probe_mixed_unique_subject_ids.shape) + #np.savetxt("probe_mixed_templates_feature.txt", probe_mixed_templates_feature) + #np.savetxt("probe_mixed_unique_subject_ids.txt", probe_mixed_unique_subject_ids) + + #root_dir = "" #feature root dir + #gallery_id_path = "" #id filepath + #gallery_feats_path = "" #feature filelpath + #print("{}: start loading gallery feat {}".format(dt.now(), gallery_id_path)) + #gallery_ids, gallery_feats = load_feat_file(root_dir, gallery_id_path, gallery_feats_path) + #print("{}: end loading gallery feat".format(dt.now())) + # + #probe_id_path = "probe_mixed_unique_subject_ids.txt" #probe id filepath + #probe_feats_path = "probe_mixed_templates_feature.txt" #probe feats filepath + #print("{}: start loading probe feat {}".format(dt.now(), probe_id_path)) + #probe_ids, probe_feats = load_feat_file(root_dir, probe_id_path, probe_feats_path) + #print("{}: end loading probe feat".format(dt.now())) + + gallery_ids = gallery_unique_subject_ids + gallery_feats = gallery_templates_feature + probe_ids = probe_mixed_unique_subject_ids + probe_feats = probe_mixed_templates_feature + + mask = gen_mask(probe_ids, gallery_ids) + + print("{}: start evaluation".format(dt.now())) + evaluation(probe_feats, gallery_feats, mask) + print("{}: end evaluation".format(dt.now())) diff --git a/Evaluation/IJB/README.md b/evaluation/IJB/README.md similarity index 100% rename from Evaluation/IJB/README.md rename to evaluation/IJB/README.md diff --git a/Evaluation/IJB/example.sh b/evaluation/IJB/example.sh similarity index 100% rename from Evaluation/IJB/example.sh rename to evaluation/IJB/example.sh diff --git a/Evaluation/IJB/readme.txt b/evaluation/IJB/readme.txt similarity index 100% rename from Evaluation/IJB/readme.txt rename to evaluation/IJB/readme.txt diff --git a/Evaluation/Megaface/README.md b/evaluation/Megaface/README.md similarity index 100% rename from Evaluation/Megaface/README.md rename to evaluation/Megaface/README.md diff --git a/evaluation/Megaface/gen_megaface.py b/evaluation/Megaface/gen_megaface.py new file mode 100644 index 0000000..2e9fdb0 --- /dev/null +++ b/evaluation/Megaface/gen_megaface.py @@ -0,0 +1,196 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os + +from easydict import EasyDict as edict +import time +import sys +import numpy as np +import argparse +import struct +import cv2 +import sklearn +from sklearn.preprocessing import normalize +import mxnet as mx +from mxnet import ndarray as nd + + +def read_img(image_path): + img = cv2.imread(image_path, cv2.CV_LOAD_IMAGE_COLOR) + return img + + +def get_feature(imgs, nets): + count = len(imgs) + data = mx.nd.zeros(shape=(count * 2, 3, imgs[0].shape[0], + imgs[0].shape[1])) + for idx, img in enumerate(imgs): + img = img[:, :, ::-1] #to rgb + img = np.transpose(img, (2, 0, 1)) + for flipid in [0, 1]: + _img = np.copy(img) + if flipid == 1: + _img = _img[:, :, ::-1] + _img = nd.array(_img) + data[count * flipid + idx] = _img + + F = [] + for net in nets: + db = mx.io.DataBatch(data=(data, )) + net.model.forward(db, is_train=False) + x = net.model.get_outputs()[0].asnumpy() + embedding = x[0:count, :] + x[count:, :] + embedding = sklearn.preprocessing.normalize(embedding) + #print('emb', embedding.shape) + F.append(embedding) + F = np.concatenate(F, axis=1) + F = sklearn.preprocessing.normalize(F) + #print('F', F.shape) + return F + + +def write_bin(path, feature): + feature = list(feature) + with open(path, 'wb') as f: + f.write(struct.pack('4i', len(feature), 1, 4, 5)) + f.write(struct.pack("%df" % len(feature), *feature)) + + +def get_and_write(buffer, nets): + imgs = [] + for k in buffer: + imgs.append(k[0]) + features = get_feature(imgs, nets) + #print(np.linalg.norm(feature)) + assert features.shape[0] == len(buffer) + for ik, k in enumerate(buffer): + out_path = k[1] + feature = features[ik].flatten() + write_bin(out_path, feature) + + +def main(args): + + print(args) + gpuid = args.gpu + ctx = mx.gpu(gpuid) + nets = [] + image_shape = [int(x) for x in args.image_size.split(',')] + for model in args.model.split('|'): + vec = model.split(',') + assert len(vec) > 1 + prefix = vec[0] + epoch = int(vec[1]) + print('loading', prefix, epoch) + net = edict() + net.ctx = ctx + net.sym, net.arg_params, net.aux_params = mx.model.load_checkpoint( + prefix, epoch) + all_layers = net.sym.get_internals() + net.sym = all_layers['fc1_output'] + net.model = mx.mod.Module(symbol=net.sym, + context=net.ctx, + label_names=None) + net.model.bind(data_shapes=[('data', (1, 3, image_shape[1], + image_shape[2]))]) + net.model.set_params(net.arg_params, net.aux_params) + nets.append(net) + + facescrub_out = os.path.join(args.output, 'facescrub') + megaface_out = os.path.join(args.output, 'megaface') + + i = 0 + succ = 0 + buffer = [] + for line in open(args.facescrub_lst, 'r'): + if i % 1000 == 0: + print("writing fs", i, succ) + i += 1 + image_path = line.strip() + _path = image_path.split('/') + a, b = _path[-2], _path[-1] + out_dir = os.path.join(facescrub_out, a) + if not os.path.exists(out_dir): + os.makedirs(out_dir) + image_path = os.path.join(args.facescrub_root, image_path) + img = read_img(image_path) + if img is None: + print('read error:', image_path) + continue + out_path = os.path.join(out_dir, b + "_%s.bin" % (args.algo)) + item = (img, out_path) + buffer.append(item) + if len(buffer) == args.batch_size: + get_and_write(buffer, nets) + buffer = [] + succ += 1 + if len(buffer) > 0: + get_and_write(buffer, nets) + buffer = [] + print('fs stat', i, succ) + + i = 0 + succ = 0 + buffer = [] + for line in open(args.megaface_lst, 'r'): + if i % 1000 == 0: + print("writing mf", i, succ) + i += 1 + image_path = line.strip() + _path = image_path.split('/') + a1, a2, b = _path[-3], _path[-2], _path[-1] + out_dir = os.path.join(megaface_out, a1, a2) + if not os.path.exists(out_dir): + os.makedirs(out_dir) + #continue + #print(landmark) + image_path = os.path.join(args.megaface_root, image_path) + img = read_img(image_path) + if img is None: + print('read error:', image_path) + continue + out_path = os.path.join(out_dir, b + "_%s.bin" % (args.algo)) + item = (img, out_path) + buffer.append(item) + if len(buffer) == args.batch_size: + get_and_write(buffer, nets) + buffer = [] + succ += 1 + if len(buffer) > 0: + get_and_write(buffer, nets) + buffer = [] + print('mf stat', i, succ) + + +def parse_arguments(argv): + parser = argparse.ArgumentParser() + + parser.add_argument('--batch_size', type=int, help='', default=8) + parser.add_argument('--image_size', type=str, help='', default='3,112,112') + parser.add_argument('--gpu', type=int, help='', default=0) + parser.add_argument('--algo', type=str, help='', default='insightface') + parser.add_argument('--facescrub-lst', + type=str, + help='', + default='./data/facescrub_lst') + parser.add_argument('--megaface-lst', + type=str, + help='', + default='./data/megaface_lst') + parser.add_argument('--facescrub-root', + type=str, + help='', + default='./data/facescrub_images') + parser.add_argument('--megaface-root', + type=str, + help='', + default='./data/megaface_images') + parser.add_argument('--output', type=str, help='', default='./feature_out') + parser.add_argument('--model', type=str, help='', default='') + return parser.parse_args(argv) + + +if __name__ == '__main__': + main(parse_arguments(sys.argv[1:])) diff --git a/evaluation/Megaface/remove_noises.py b/evaluation/Megaface/remove_noises.py new file mode 100644 index 0000000..aacec54 --- /dev/null +++ b/evaluation/Megaface/remove_noises.py @@ -0,0 +1,182 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import datetime +import time +import shutil +import sys +import numpy as np +import argparse +import struct +import cv2 +import mxnet as mx +from mxnet import ndarray as nd + +feature_dim = 512 +feature_ext = 1 + + +def load_bin(path, fill=0.0): + with open(path, 'rb') as f: + bb = f.read(4 * 4) + #print(len(bb)) + v = struct.unpack('4i', bb) + #print(v[0]) + bb = f.read(v[0] * 4) + v = struct.unpack("%df" % (v[0]), bb) + feature = np.full((feature_dim + feature_ext, ), + fill, + dtype=np.float32) + feature[0:feature_dim] = v + #feature = np.array( v, dtype=np.float32) + #print(feature.shape) + #print(np.linalg.norm(feature)) + return feature + + +def write_bin(path, feature): + feature = list(feature) + with open(path, 'wb') as f: + f.write(struct.pack('4i', len(feature), 1, 4, 5)) + f.write(struct.pack("%df" % len(feature), *feature)) + + +def main(args): + + fs_noise_map = {} + for line in open(args.facescrub_noises, 'r'): + if line.startswith('#'): + continue + line = line.strip() + fname = line.split('.')[0] + p = fname.rfind('_') + fname = fname[0:p] + fs_noise_map[line] = fname + + print(len(fs_noise_map)) + + i = 0 + fname2center = {} + noises = [] + for line in open(args.facescrub_lst, 'r'): + if i % 1000 == 0: + print("reading fs", i) + i += 1 + image_path = line.strip() + _path = image_path.split('/') + a, b = _path[-2], _path[-1] + feature_path = os.path.join(args.feature_dir_input, 'facescrub', a, + "%s_%s.bin" % (b, args.algo)) + feature_dir_out = os.path.join(args.feature_dir_out, 'facescrub', a) + if not os.path.exists(feature_dir_out): + os.makedirs(feature_dir_out) + feature_path_out = os.path.join(feature_dir_out, + "%s_%s.bin" % (b, args.algo)) + #print(b) + if not b in fs_noise_map: + #shutil.copyfile(feature_path, feature_path_out) + feature = load_bin(feature_path) + write_bin(feature_path_out, feature) + if not a in fname2center: + fname2center[a] = np.zeros((feature_dim + feature_ext, ), + dtype=np.float32) + fname2center[a] += feature + else: + #print('n', b) + noises.append((a, b)) + print(len(noises)) + + for k in noises: + a, b = k + assert a in fname2center + center = fname2center[a] + g = np.zeros((feature_dim + feature_ext, ), dtype=np.float32) + g2 = np.random.uniform(-0.001, 0.001, (feature_dim, )) + g[0:feature_dim] = g2 + f = center + g + _norm = np.linalg.norm(f) + f /= _norm + feature_path_out = os.path.join(args.feature_dir_out, 'facescrub', a, + "%s_%s.bin" % (b, args.algo)) + write_bin(feature_path_out, f) + + mf_noise_map = {} + for line in open(args.megaface_noises, 'r'): + if line.startswith('#'): + continue + line = line.strip() + _vec = line.split("\t") + if len(_vec) > 1: + line = _vec[1] + mf_noise_map[line] = 1 + + print(len(mf_noise_map)) + + i = 0 + nrof_noises = 0 + for line in open(args.megaface_lst, 'r'): + if i % 1000 == 0: + print("reading mf", i) + i += 1 + image_path = line.strip() + _path = image_path.split('/') + a1, a2, b = _path[-3], _path[-2], _path[-1] + feature_path = os.path.join(args.feature_dir_input, 'megaface', a1, a2, + "%s_%s.bin" % (b, args.algo)) + feature_dir_out = os.path.join(args.feature_dir_out, 'megaface', a1, + a2) + if not os.path.exists(feature_dir_out): + os.makedirs(feature_dir_out) + feature_path_out = os.path.join(feature_dir_out, + "%s_%s.bin" % (b, args.algo)) + bb = '/'.join([a1, a2, b]) + #print(b) + if not bb in mf_noise_map: + feature = load_bin(feature_path) + write_bin(feature_path_out, feature) + #shutil.copyfile(feature_path, feature_path_out) + else: + feature = load_bin(feature_path, 100.0) + write_bin(feature_path_out, feature) + #g = np.random.uniform(-0.001, 0.001, (feature_dim,)) + #print('n', bb) + #write_bin(feature_path_out, g) + nrof_noises += 1 + print(nrof_noises) + + +def parse_arguments(argv): + parser = argparse.ArgumentParser() + + parser.add_argument('--facescrub-noises', + type=str, + help='', + default='./data/facescrub_noises.txt') + parser.add_argument('--megaface-noises', + type=str, + help='', + default='./data/megaface_noises.txt') + parser.add_argument('--algo', type=str, help='', default='insightface') + parser.add_argument('--facescrub-lst', + type=str, + help='', + default='./data/facescrub_lst') + parser.add_argument('--megaface-lst', + type=str, + help='', + default='./data/megaface_lst') + parser.add_argument('--feature-dir-input', + type=str, + help='', + default='./feature_out') + parser.add_argument('--feature-dir-out', + type=str, + help='', + default='./feature_out_clean') + return parser.parse_args(argv) + + +if __name__ == '__main__': + main(parse_arguments(sys.argv[1:])) diff --git a/Evaluation/Megaface/run.sh b/evaluation/Megaface/run.sh similarity index 100% rename from Evaluation/Megaface/run.sh rename to evaluation/Megaface/run.sh diff --git a/gender-age/data.py b/gender-age/data.py index 21441f4..103728e 100644 --- a/gender-age/data.py +++ b/gender-age/data.py @@ -24,17 +24,23 @@ logger = logging.getLogger() class FaceImageIter(io.DataIter): - - def __init__(self, batch_size, data_shape, - path_imgrec = None, - shuffle=False, aug_list=None, mean = None, - rand_mirror = False, cutoff = 0, color_jittering = 0, - data_name='data', label_name='softmax_label', **kwargs): + def __init__(self, + batch_size, + data_shape, + path_imgrec=None, + shuffle=False, + aug_list=None, + mean=None, + rand_mirror=False, + cutoff=0, + color_jittering=0, + data_name='data', + label_name='softmax_label', + **kwargs): super(FaceImageIter, self).__init__() assert path_imgrec - logging.info('loading recordio %s...', - path_imgrec) - path_imgidx = path_imgrec[0:-4]+".idx" + logging.info('loading recordio %s...', path_imgrec) + path_imgidx = path_imgrec[0:-4] + ".idx" self.imgrec = recordio.MXIndexedRecordIO(path_imgidx, path_imgrec, 'r') # pylint: disable=redefined-variable-type s = self.imgrec.read_idx(0) header, _ = recordio.unpack(s) @@ -44,38 +50,37 @@ class FaceImageIter(io.DataIter): self.mean = mean self.nd_mean = None if self.mean: - self.mean = np.array(self.mean, dtype=np.float32).reshape(1,1,3) - self.nd_mean = mx.nd.array(self.mean).reshape((1,1,3)) + self.mean = np.array(self.mean, dtype=np.float32).reshape(1, 1, 3) + self.nd_mean = mx.nd.array(self.mean).reshape((1, 1, 3)) self.check_data_shape(data_shape) - self.provide_data = [(data_name, (batch_size,) + data_shape)] + self.provide_data = [(data_name, (batch_size, ) + data_shape)] self.batch_size = batch_size self.data_shape = data_shape self.shuffle = shuffle - self.image_size = '%d,%d'%(data_shape[1],data_shape[2]) + self.image_size = '%d,%d' % (data_shape[1], data_shape[2]) self.rand_mirror = rand_mirror print('rand_mirror', rand_mirror) self.cutoff = cutoff self.color_jittering = color_jittering self.CJA = mx.image.ColorJitterAug(0.125, 0.125, 0.125) - self.provide_label = [(label_name, (batch_size,101))] + self.provide_label = [(label_name, (batch_size, 101))] #print(self.provide_label[0][1]) self.cur = 0 self.nbatch = 0 self.is_init = False - def reset(self): """Resets the iterator to the beginning of the data.""" print('call reset()') self.cur = 0 if self.shuffle: - random.shuffle(self.seq) + random.shuffle(self.seq) if self.seq is None and self.imgrec is not None: self.imgrec.reset() def num_samples(self): - return len(self.seq) + return len(self.seq) def next_sample(self): if self.cur >= len(self.seq): @@ -88,68 +93,67 @@ class FaceImageIter(io.DataIter): return label, img, None, None def brightness_aug(self, src, x): - alpha = 1.0 + random.uniform(-x, x) - src *= alpha - return src + alpha = 1.0 + random.uniform(-x, x) + src *= alpha + return src def contrast_aug(self, src, x): - alpha = 1.0 + random.uniform(-x, x) - coef = nd.array([[[0.299, 0.587, 0.114]]]) - gray = src * coef - gray = (3.0 * (1.0 - alpha) / gray.size) * nd.sum(gray) - src *= alpha - src += gray - return src + alpha = 1.0 + random.uniform(-x, x) + coef = nd.array([[[0.299, 0.587, 0.114]]]) + gray = src * coef + gray = (3.0 * (1.0 - alpha) / gray.size) * nd.sum(gray) + src *= alpha + src += gray + return src def saturation_aug(self, src, x): - alpha = 1.0 + random.uniform(-x, x) - coef = nd.array([[[0.299, 0.587, 0.114]]]) - gray = src * coef - gray = nd.sum(gray, axis=2, keepdims=True) - gray *= (1.0 - alpha) - src *= alpha - src += gray - return src + alpha = 1.0 + random.uniform(-x, x) + coef = nd.array([[[0.299, 0.587, 0.114]]]) + gray = src * coef + gray = nd.sum(gray, axis=2, keepdims=True) + gray *= (1.0 - alpha) + src *= alpha + src += gray + return src def color_aug(self, img, x): - #augs = [self.brightness_aug, self.contrast_aug, self.saturation_aug] - #random.shuffle(augs) - #for aug in augs: - # #print(img.shape) - # img = aug(img, x) - # #print(img.shape) - #return img - return self.CJA(img) + #augs = [self.brightness_aug, self.contrast_aug, self.saturation_aug] + #random.shuffle(augs) + #for aug in augs: + # #print(img.shape) + # img = aug(img, x) + # #print(img.shape) + #return img + return self.CJA(img) def mirror_aug(self, img): - _rd = random.randint(0,1) - if _rd==1: - for c in range(img.shape[2]): - img[:,:,c] = np.fliplr(img[:,:,c]) - return img + _rd = random.randint(0, 1) + if _rd == 1: + for c in range(img.shape[2]): + img[:, :, c] = np.fliplr(img[:, :, c]) + return img def compress_aug(self, img): - buf = BytesIO() - img = Image.fromarray(img.asnumpy(), 'RGB') - q = random.randint(2, 20) - img.save(buf, format='JPEG', quality=q) - buf = buf.getvalue() - img = Image.open(BytesIO(buf)) - return nd.array(np.asarray(img, 'float32')) - + buf = BytesIO() + img = Image.fromarray(img.asnumpy(), 'RGB') + q = random.randint(2, 20) + img.save(buf, format='JPEG', quality=q) + buf = buf.getvalue() + img = Image.open(BytesIO(buf)) + return nd.array(np.asarray(img, 'float32')) def next(self): if not self.is_init: - self.reset() - self.is_init = True + self.reset() + self.is_init = True """Returns the next batch of data.""" #print('in next', self.cur, self.labelcur) - self.nbatch+=1 + self.nbatch += 1 batch_size = self.batch_size c, h, w = self.data_shape batch_data = nd.empty((batch_size, c, h, w)) if self.provide_label is not None: - batch_label = nd.empty(self.provide_label[0][1]) + batch_label = nd.empty(self.provide_label[0][1]) i = 0 try: while i < batch_size: @@ -157,49 +161,49 @@ class FaceImageIter(io.DataIter): label, s, bbox, landmark = self.next_sample() gender = int(label[0]) age = int(label[1]) - assert age>=0 + assert age >= 0 #assert gender==0 or gender==1 - plabel = np.zeros(shape=(101,), dtype=np.float32) + plabel = np.zeros(shape=(101, ), dtype=np.float32) plabel[0] = gender - if age==0: - age = 1 - if age>100: - age = 100 - plabel[1:age+1] = 1 + if age == 0: + age = 1 + if age > 100: + age = 100 + plabel[1:age + 1] = 1 label = plabel _data = self.imdecode(s) - if _data.shape[0]!=self.data_shape[1]: - _data = mx.image.resize_short(_data, self.data_shape[1]) + if _data.shape[0] != self.data_shape[1]: + _data = mx.image.resize_short(_data, self.data_shape[1]) if self.rand_mirror: - _rd = random.randint(0,1) - if _rd==1: - _data = mx.ndarray.flip(data=_data, axis=1) - if self.color_jittering>0: - if self.color_jittering>1: - _rd = random.randint(0,1) - if _rd==1: - _data = self.compress_aug(_data) - #print('do color aug') - _data = _data.astype('float32', copy=False) - #print(_data.__class__) - _data = self.color_aug(_data, 0.125) + _rd = random.randint(0, 1) + if _rd == 1: + _data = mx.ndarray.flip(data=_data, axis=1) + if self.color_jittering > 0: + if self.color_jittering > 1: + _rd = random.randint(0, 1) + if _rd == 1: + _data = self.compress_aug(_data) + #print('do color aug') + _data = _data.astype('float32', copy=False) + #print(_data.__class__) + _data = self.color_aug(_data, 0.125) if self.nd_mean is not None: - _data = _data.astype('float32', copy=False) - _data -= self.nd_mean - _data *= 0.0078125 - if self.cutoff>0: - _rd = random.randint(0,1) - if _rd==1: - #print('do cutoff aug', self.cutoff) - centerh = random.randint(0, _data.shape[0]-1) - centerw = random.randint(0, _data.shape[1]-1) - half = self.cutoff//2 - starth = max(0, centerh-half) - endh = min(_data.shape[0], centerh+half) - startw = max(0, centerw-half) - endw = min(_data.shape[1], centerw+half) - #print(starth, endh, startw, endw, _data.shape) - _data[starth:endh, startw:endw, :] = 128 + _data = _data.astype('float32', copy=False) + _data -= self.nd_mean + _data *= 0.0078125 + if self.cutoff > 0: + _rd = random.randint(0, 1) + if _rd == 1: + #print('do cutoff aug', self.cutoff) + centerh = random.randint(0, _data.shape[0] - 1) + centerw = random.randint(0, _data.shape[1] - 1) + half = self.cutoff // 2 + starth = max(0, centerh - half) + endh = min(_data.shape[0], centerh + half) + startw = max(0, centerw - half) + endw = min(_data.shape[1], centerw + half) + #print(starth, endh, startw, endw, _data.shape) + _data[starth:endh, startw:endw, :] = 128 data = [_data] for datum in data: assert i < batch_size, 'Batch size must be multiples of augmenter output length' @@ -208,7 +212,7 @@ class FaceImageIter(io.DataIter): batch_label[i][:] = label i += 1 except StopIteration: - if i0 - self.provide_data = iter_list[0].provide_data - self.provide_label = iter_list[0].provide_label - self.iter_list = iter_list - self.cur_iter = None + def __init__(self, iter_list): + assert len(iter_list) > 0 + self.provide_data = iter_list[0].provide_data + self.provide_label = iter_list[0].provide_label + self.iter_list = iter_list + self.cur_iter = None - def reset(self): - self.cur_iter.reset() - - def next(self): - self.cur_iter = random.choice(self.iter_list) - while True: - try: - ret = self.cur_iter.next() - except StopIteration: + def reset(self): self.cur_iter.reset() - continue - return ret - + def next(self): + self.cur_iter = random.choice(self.iter_list) + while True: + try: + ret = self.cur_iter.next() + except StopIteration: + self.cur_iter.reset() + continue + return ret diff --git a/gender-age/face_model.py b/gender-age/face_model.py index 1212a25..1ce54bb 100644 --- a/gender-age/face_model.py +++ b/gender-age/face_model.py @@ -22,78 +22,88 @@ import face_preprocess def do_flip(data): - for idx in range(data.shape[0]): - data[idx,:,:] = np.fliplr(data[idx,:,:]) + for idx in range(data.shape[0]): + data[idx, :, :] = np.fliplr(data[idx, :, :]) + def get_model(ctx, image_size, model_str, layer): - _vec = model_str.split(',') - assert len(_vec)==2 - prefix = _vec[0] - epoch = int(_vec[1]) - print('loading',prefix, epoch) - sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) - all_layers = sym.get_internals() - sym = all_layers[layer+'_output'] - model = mx.mod.Module(symbol=sym, context=ctx, label_names = None) - #model.bind(data_shapes=[('data', (args.batch_size, 3, image_size[0], image_size[1]))], label_shapes=[('softmax_label', (args.batch_size,))]) - model.bind(data_shapes=[('data', (1, 3, image_size[0], image_size[1]))]) - model.set_params(arg_params, aux_params) - return model + _vec = model_str.split(',') + assert len(_vec) == 2 + prefix = _vec[0] + epoch = int(_vec[1]) + print('loading', prefix, epoch) + sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) + all_layers = sym.get_internals() + sym = all_layers[layer + '_output'] + model = mx.mod.Module(symbol=sym, context=ctx, label_names=None) + #model.bind(data_shapes=[('data', (args.batch_size, 3, image_size[0], image_size[1]))], label_shapes=[('softmax_label', (args.batch_size,))]) + model.bind(data_shapes=[('data', (1, 3, image_size[0], image_size[1]))]) + model.set_params(arg_params, aux_params) + return model + class FaceModel: - def __init__(self, args): - self.args = args - if args.gpu>=0: - ctx = mx.gpu(args.gpu) - else: - ctx = mx.cpu() - _vec = args.image_size.split(',') - assert len(_vec)==2 - image_size = (int(_vec[0]), int(_vec[1])) - self.model = None - if len(args.model)>0: - self.model = get_model(ctx, image_size, args.model, 'fc1') + def __init__(self, args): + self.args = args + if args.gpu >= 0: + ctx = mx.gpu(args.gpu) + else: + ctx = mx.cpu() + _vec = args.image_size.split(',') + assert len(_vec) == 2 + image_size = (int(_vec[0]), int(_vec[1])) + self.model = None + if len(args.model) > 0: + self.model = get_model(ctx, image_size, args.model, 'fc1') - self.det_minsize = 50 - self.det_threshold = [0.6,0.7,0.8] - #self.det_factor = 0.9 - self.image_size = image_size - mtcnn_path = os.path.join(os.path.dirname(__file__), 'mtcnn-model') - if args.det==0: - detector = MtcnnDetector(model_folder=mtcnn_path, ctx=ctx, num_worker=1, accurate_landmark = True, threshold=self.det_threshold) - else: - detector = MtcnnDetector(model_folder=mtcnn_path, ctx=ctx, num_worker=1, accurate_landmark = True, threshold=[0.0,0.0,0.2]) - self.detector = detector + self.det_minsize = 50 + self.det_threshold = [0.6, 0.7, 0.8] + #self.det_factor = 0.9 + self.image_size = image_size + mtcnn_path = os.path.join(os.path.dirname(__file__), 'mtcnn-model') + if args.det == 0: + detector = MtcnnDetector(model_folder=mtcnn_path, + ctx=ctx, + num_worker=1, + accurate_landmark=True, + threshold=self.det_threshold) + else: + detector = MtcnnDetector(model_folder=mtcnn_path, + ctx=ctx, + num_worker=1, + accurate_landmark=True, + threshold=[0.0, 0.0, 0.2]) + self.detector = detector + def get_input(self, face_img): + ret = self.detector.detect_face(face_img, det_type=self.args.det) + if ret is None: + return None + bbox, points = ret + if bbox.shape[0] == 0: + return None + bbox = bbox[0, 0:4] + points = points[0, :].reshape((2, 5)).T + #print(bbox) + #print(points) + nimg = face_preprocess.preprocess(face_img, + bbox, + points, + image_size='112,112') + nimg = cv2.cvtColor(nimg, cv2.COLOR_BGR2RGB) + aligned = np.transpose(nimg, (2, 0, 1)) + input_blob = np.expand_dims(aligned, axis=0) + data = mx.nd.array(input_blob) + db = mx.io.DataBatch(data=(data, )) + return db - def get_input(self, face_img): - ret = self.detector.detect_face(face_img, det_type = self.args.det) - if ret is None: - return None - bbox, points = ret - if bbox.shape[0]==0: - return None - bbox = bbox[0,0:4] - points = points[0,:].reshape((2,5)).T - #print(bbox) - #print(points) - nimg = face_preprocess.preprocess(face_img, bbox, points, image_size='112,112') - nimg = cv2.cvtColor(nimg, cv2.COLOR_BGR2RGB) - aligned = np.transpose(nimg, (2,0,1)) - input_blob = np.expand_dims(aligned, axis=0) - data = mx.nd.array(input_blob) - db = mx.io.DataBatch(data=(data,)) - return db - - - def get_ga(self, data): - self.model.forward(data, is_train=False) - ret = self.model.get_outputs()[0].asnumpy() - g = ret[:,0:2].flatten() - gender = np.argmax(g) - a = ret[:,2:202].reshape( (100,2) ) - a = np.argmax(a, axis=1) - age = int(sum(a)) - - return gender, age + def get_ga(self, data): + self.model.forward(data, is_train=False) + ret = self.model.get_outputs()[0].asnumpy() + g = ret[:, 0:2].flatten() + gender = np.argmax(g) + a = ret[:, 2:202].reshape((100, 2)) + a = np.argmax(a, axis=1) + age = int(sum(a)) + return gender, age diff --git a/gender-age/fmobilenet.py b/gender-age/fmobilenet.py index d2bdeb8..6bb1c0a 100644 --- a/gender-age/fmobilenet.py +++ b/gender-age/fmobilenet.py @@ -18,68 +18,254 @@ import mxnet as mx import symbol_utils + def Act(data, act_type, name): - #ignore param act_type, set it in this function + #ignore param act_type, set it in this function #body = mx.sym.LeakyReLU(data = data, act_type='prelu', name = name) body = mx.sym.Activation(data=data, act_type='relu', name=name) return body -def Conv(data, num_filter=1, kernel=(1, 1), stride=(1, 1), pad=(0, 0), num_group=1, name=None, suffix=''): - conv = mx.sym.Convolution(data=data, num_filter=num_filter, kernel=kernel, num_group=num_group, stride=stride, pad=pad, no_bias=True, name='%s%s_conv2d' %(name, suffix)) - bn = mx.sym.BatchNorm(data=conv, name='%s%s_batchnorm' %(name, suffix), fix_gamma=True) - act = Act(data=bn, act_type='relu', name='%s%s_relu' %(name, suffix)) + +def Conv(data, + num_filter=1, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + num_group=1, + name=None, + suffix=''): + conv = mx.sym.Convolution(data=data, + num_filter=num_filter, + kernel=kernel, + num_group=num_group, + stride=stride, + pad=pad, + no_bias=True, + name='%s%s_conv2d' % (name, suffix)) + bn = mx.sym.BatchNorm(data=conv, + name='%s%s_batchnorm' % (name, suffix), + fix_gamma=True) + act = Act(data=bn, act_type='relu', name='%s%s_relu' % (name, suffix)) return act -def ConvOnly(data, num_filter=1, kernel=(1, 1), stride=(1, 1), pad=(0, 0), num_group=1, name=None, suffix=''): - conv = mx.sym.Convolution(data=data, num_filter=num_filter, kernel=kernel, num_group=num_group, stride=stride, pad=pad, no_bias=True, name='%s%s_conv2d' %(name, suffix)) + +def ConvOnly(data, + num_filter=1, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + num_group=1, + name=None, + suffix=''): + conv = mx.sym.Convolution(data=data, + num_filter=num_filter, + kernel=kernel, + num_group=num_group, + stride=stride, + pad=pad, + no_bias=True, + name='%s%s_conv2d' % (name, suffix)) return conv + def get_symbol(num_classes, **kwargs): - data = mx.symbol.Variable(name="data") # 224 - data = data-127.5 - data = data*0.0078125 + data = mx.symbol.Variable(name="data") # 224 + data = data - 127.5 + data = data * 0.0078125 version_input = kwargs.get('version_input', 1) - assert version_input>=0 + assert version_input >= 0 version_output = kwargs.get('version_output', 'E') multiplier = kwargs.get('multiplier', 1.0) fc_type = version_output - base_filter = int(32*multiplier) + base_filter = int(32 * multiplier) bf = base_filter print(version_input, version_output, base_filter) - if version_input==0: - conv_1 = Conv(data, num_filter=bf, kernel=(3, 3), pad=(1, 1), stride=(2, 2), name="conv_1") # 224/112 + if version_input == 0: + conv_1 = Conv(data, + num_filter=bf, + kernel=(3, 3), + pad=(1, 1), + stride=(2, 2), + name="conv_1") # 224/112 else: - conv_1 = Conv(data, num_filter=bf, kernel=(3, 3), pad=(1, 1), stride=(1, 1), name="conv_1") # 224/112 - conv_2_dw = Conv(conv_1, num_group=bf, num_filter=bf, kernel=(3, 3), pad=(1, 1), stride=(1, 1), name="conv_2_dw") # 112/112 - conv_2 = Conv(conv_2_dw, num_filter=bf*2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_2") # 112/112 - conv_3_dw = Conv(conv_2, num_group=bf*2, num_filter=bf*2, kernel=(3, 3), pad=(1, 1), stride=(2, 2), name="conv_3_dw") # 112/56 - conv_3 = Conv(conv_3_dw, num_filter=bf*4, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_3") # 56/56 - conv_4_dw = Conv(conv_3, num_group=bf*4, num_filter=bf*4, kernel=(3, 3), pad=(1, 1), stride=(1, 1), name="conv_4_dw") # 56/56 - conv_4 = Conv(conv_4_dw, num_filter=bf*4, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_4") # 56/56 - conv_5_dw = Conv(conv_4, num_group=bf*4, num_filter=bf*4, kernel=(3, 3), pad=(1, 1), stride=(2, 2), name="conv_5_dw") # 56/28 - conv_5 = Conv(conv_5_dw, num_filter=bf*8, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_5") # 28/28 - conv_6_dw = Conv(conv_5, num_group=bf*8, num_filter=bf*8, kernel=(3, 3), pad=(1, 1), stride=(1, 1), name="conv_6_dw") # 28/28 - conv_6 = Conv(conv_6_dw, num_filter=bf*8, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_6") # 28/28 - conv_7_dw = Conv(conv_6, num_group=bf*8, num_filter=bf*8, kernel=(3, 3), pad=(1, 1), stride=(2, 2), name="conv_7_dw") # 28/14 - conv_7 = Conv(conv_7_dw, num_filter=bf*16, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_7") # 14/14 + conv_1 = Conv(data, + num_filter=bf, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + name="conv_1") # 224/112 + conv_2_dw = Conv(conv_1, + num_group=bf, + num_filter=bf, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + name="conv_2_dw") # 112/112 + conv_2 = Conv(conv_2_dw, + num_filter=bf * 2, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + name="conv_2") # 112/112 + conv_3_dw = Conv(conv_2, + num_group=bf * 2, + num_filter=bf * 2, + kernel=(3, 3), + pad=(1, 1), + stride=(2, 2), + name="conv_3_dw") # 112/56 + conv_3 = Conv(conv_3_dw, + num_filter=bf * 4, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + name="conv_3") # 56/56 + conv_4_dw = Conv(conv_3, + num_group=bf * 4, + num_filter=bf * 4, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + name="conv_4_dw") # 56/56 + conv_4 = Conv(conv_4_dw, + num_filter=bf * 4, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + name="conv_4") # 56/56 + conv_5_dw = Conv(conv_4, + num_group=bf * 4, + num_filter=bf * 4, + kernel=(3, 3), + pad=(1, 1), + stride=(2, 2), + name="conv_5_dw") # 56/28 + conv_5 = Conv(conv_5_dw, + num_filter=bf * 8, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + name="conv_5") # 28/28 + conv_6_dw = Conv(conv_5, + num_group=bf * 8, + num_filter=bf * 8, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + name="conv_6_dw") # 28/28 + conv_6 = Conv(conv_6_dw, + num_filter=bf * 8, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + name="conv_6") # 28/28 + conv_7_dw = Conv(conv_6, + num_group=bf * 8, + num_filter=bf * 8, + kernel=(3, 3), + pad=(1, 1), + stride=(2, 2), + name="conv_7_dw") # 28/14 + conv_7 = Conv(conv_7_dw, + num_filter=bf * 16, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + name="conv_7") # 14/14 - conv_8_dw = Conv(conv_7, num_group=bf*16, num_filter=bf*16, kernel=(3, 3), pad=(1, 1), stride=(1, 1), name="conv_8_dw") # 14/14 - conv_8 = Conv(conv_8_dw, num_filter=bf*16, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_8") # 14/14 - conv_9_dw = Conv(conv_8, num_group=bf*16, num_filter=bf*16, kernel=(3, 3), pad=(1, 1), stride=(1, 1), name="conv_9_dw") # 14/14 - conv_9 = Conv(conv_9_dw, num_filter=bf*16, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_9") # 14/14 - conv_10_dw = Conv(conv_9, num_group=bf*16, num_filter=bf*16, kernel=(3, 3), pad=(1, 1), stride=(1, 1), name="conv_10_dw") # 14/14 - conv_10 = Conv(conv_10_dw, num_filter=bf*16, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_10") # 14/14 - conv_11_dw = Conv(conv_10, num_group=bf*16, num_filter=bf*16, kernel=(3, 3), pad=(1, 1), stride=(1, 1), name="conv_11_dw") # 14/14 - conv_11 = Conv(conv_11_dw, num_filter=bf*16, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_11") # 14/14 - conv_12_dw = Conv(conv_11, num_group=bf*16, num_filter=bf*16, kernel=(3, 3), pad=(1, 1), stride=(1, 1), name="conv_12_dw") # 14/14 - conv_12 = Conv(conv_12_dw, num_filter=bf*16, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_12") # 14/14 + conv_8_dw = Conv(conv_7, + num_group=bf * 16, + num_filter=bf * 16, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + name="conv_8_dw") # 14/14 + conv_8 = Conv(conv_8_dw, + num_filter=bf * 16, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + name="conv_8") # 14/14 + conv_9_dw = Conv(conv_8, + num_group=bf * 16, + num_filter=bf * 16, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + name="conv_9_dw") # 14/14 + conv_9 = Conv(conv_9_dw, + num_filter=bf * 16, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + name="conv_9") # 14/14 + conv_10_dw = Conv(conv_9, + num_group=bf * 16, + num_filter=bf * 16, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + name="conv_10_dw") # 14/14 + conv_10 = Conv(conv_10_dw, + num_filter=bf * 16, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + name="conv_10") # 14/14 + conv_11_dw = Conv(conv_10, + num_group=bf * 16, + num_filter=bf * 16, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + name="conv_11_dw") # 14/14 + conv_11 = Conv(conv_11_dw, + num_filter=bf * 16, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + name="conv_11") # 14/14 + conv_12_dw = Conv(conv_11, + num_group=bf * 16, + num_filter=bf * 16, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + name="conv_12_dw") # 14/14 + conv_12 = Conv(conv_12_dw, + num_filter=bf * 16, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + name="conv_12") # 14/14 - conv_13_dw = Conv(conv_12, num_group=bf*16, num_filter=bf*16, kernel=(3, 3), pad=(1, 1), stride=(2, 2), name="conv_13_dw") # 14/7 - conv_13 = Conv(conv_13_dw, num_filter=bf*32, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_13") # 7/7 - conv_14_dw = Conv(conv_13, num_group=bf*32, num_filter=bf*32, kernel=(3, 3), pad=(1, 1), stride=(1, 1), name="conv_14_dw") # 7/7 - conv_14 = Conv(conv_14_dw, num_filter=bf*32, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_14") # 7/7 + conv_13_dw = Conv(conv_12, + num_group=bf * 16, + num_filter=bf * 16, + kernel=(3, 3), + pad=(1, 1), + stride=(2, 2), + name="conv_13_dw") # 14/7 + conv_13 = Conv(conv_13_dw, + num_filter=bf * 32, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + name="conv_13") # 7/7 + conv_14_dw = Conv(conv_13, + num_group=bf * 32, + num_filter=bf * 32, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + name="conv_14_dw") # 7/7 + conv_14 = Conv(conv_14_dw, + num_filter=bf * 32, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + name="conv_14") # 7/7 body = conv_14 fc1 = symbol_utils.get_fc1(body, num_classes, fc_type) return fc1 - diff --git a/gender-age/fresnet.py b/gender-age/fresnet.py index d4844b7..037a42b 100644 --- a/gender-age/fresnet.py +++ b/gender-age/fresnet.py @@ -14,7 +14,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - ''' Adapted from https://github.com/tornadomeet/ResNet/blob/master/symbol_resnet.py Original author Wei Wu @@ -31,6 +30,7 @@ import numpy as np import symbol_utils import sklearn + def Conv(**kwargs): #name = kwargs.get('name') #_weight = mx.symbol.Variable(name+'_weight') @@ -41,13 +41,15 @@ def Conv(**kwargs): def Act(data, act_type, name): - if act_type=='prelu': - body = mx.sym.LeakyReLU(data = data, act_type='prelu', name = name) + if act_type == 'prelu': + body = mx.sym.LeakyReLU(data=data, act_type='prelu', name=name) else: - body = mx.symbol.Activation(data=data, act_type=act_type, name=name) + body = mx.symbol.Activation(data=data, act_type=act_type, name=name) return body -def residual_unit_v1(data, num_filter, stride, dim_match, name, bottle_neck, **kwargs): + +def residual_unit_v1(data, num_filter, stride, dim_match, name, bottle_neck, + **kwargs): """Return ResNet Unit symbol for building ResNet Parameters ---------- @@ -73,70 +75,176 @@ def residual_unit_v1(data, num_filter, stride, dim_match, name, bottle_neck, **k act_type = kwargs.get('version_act', 'prelu') #print('in unit1') if bottle_neck: - conv1 = Conv(data=data, num_filter=int(num_filter*0.25), kernel=(1,1), stride=stride, pad=(0,0), - no_bias=True, workspace=workspace, name=name + '_conv1') - bn1 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn1') + conv1 = Conv(data=data, + num_filter=int(num_filter * 0.25), + kernel=(1, 1), + stride=stride, + pad=(0, 0), + no_bias=True, + workspace=workspace, + name=name + '_conv1') + bn1 = mx.sym.BatchNorm(data=conv1, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn1') act1 = Act(data=bn1, act_type=act_type, name=name + '_relu1') - conv2 = Conv(data=act1, num_filter=int(num_filter*0.25), kernel=(3,3), stride=(1,1), pad=(1,1), - no_bias=True, workspace=workspace, name=name + '_conv2') - bn2 = mx.sym.BatchNorm(data=conv2, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn2') + conv2 = Conv(data=act1, + num_filter=int(num_filter * 0.25), + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv2') + bn2 = mx.sym.BatchNorm(data=conv2, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn2') act2 = Act(data=bn2, act_type=act_type, name=name + '_relu2') - conv3 = Conv(data=act2, num_filter=num_filter, kernel=(1,1), stride=(1,1), pad=(0,0), no_bias=True, - workspace=workspace, name=name + '_conv3') - bn3 = mx.sym.BatchNorm(data=conv3, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn3') + conv3 = Conv(data=act2, + num_filter=num_filter, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + no_bias=True, + workspace=workspace, + name=name + '_conv3') + bn3 = mx.sym.BatchNorm(data=conv3, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn3') if use_se: - #se begin - body = mx.sym.Pooling(data=bn3, global_pool=True, kernel=(7, 7), pool_type='avg', name=name+'_se_pool1') - body = Conv(data=body, num_filter=num_filter//16, kernel=(1,1), stride=(1,1), pad=(0,0), - name=name+"_se_conv1", workspace=workspace) - body = Act(data=body, act_type=act_type, name=name+'_se_relu1') - body = Conv(data=body, num_filter=num_filter, kernel=(1,1), stride=(1,1), pad=(0,0), - name=name+"_se_conv2", workspace=workspace) - body = mx.symbol.Activation(data=body, act_type='sigmoid', name=name+"_se_sigmoid") - bn3 = mx.symbol.broadcast_mul(bn3, body) - #se end + #se begin + body = mx.sym.Pooling(data=bn3, + global_pool=True, + kernel=(7, 7), + pool_type='avg', + name=name + '_se_pool1') + body = Conv(data=body, + num_filter=num_filter // 16, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv1", + workspace=workspace) + body = Act(data=body, act_type=act_type, name=name + '_se_relu1') + body = Conv(data=body, + num_filter=num_filter, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv2", + workspace=workspace) + body = mx.symbol.Activation(data=body, + act_type='sigmoid', + name=name + "_se_sigmoid") + bn3 = mx.symbol.broadcast_mul(bn3, body) + #se end if dim_match: shortcut = data else: - conv1sc = Conv(data=data, num_filter=num_filter, kernel=(1,1), stride=stride, no_bias=True, - workspace=workspace, name=name+'_conv1sc') - shortcut = mx.sym.BatchNorm(data=conv1sc, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_sc') + conv1sc = Conv(data=data, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + no_bias=True, + workspace=workspace, + name=name + '_conv1sc') + shortcut = mx.sym.BatchNorm(data=conv1sc, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_sc') if memonger: shortcut._set_attr(mirror_stage='True') - return Act(data=bn3 + shortcut, act_type=act_type, name=name + '_relu3') + return Act(data=bn3 + shortcut, + act_type=act_type, + name=name + '_relu3') else: - conv1 = Conv(data=data, num_filter=num_filter, kernel=(3,3), stride=stride, pad=(1,1), - no_bias=True, workspace=workspace, name=name + '_conv1') - bn1 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_bn1') + conv1 = Conv(data=data, + num_filter=num_filter, + kernel=(3, 3), + stride=stride, + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv1') + bn1 = mx.sym.BatchNorm(data=conv1, + fix_gamma=False, + momentum=bn_mom, + eps=2e-5, + name=name + '_bn1') act1 = Act(data=bn1, act_type=act_type, name=name + '_relu1') - conv2 = Conv(data=act1, num_filter=num_filter, kernel=(3,3), stride=(1,1), pad=(1,1), - no_bias=True, workspace=workspace, name=name + '_conv2') - bn2 = mx.sym.BatchNorm(data=conv2, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_bn2') + conv2 = Conv(data=act1, + num_filter=num_filter, + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv2') + bn2 = mx.sym.BatchNorm(data=conv2, + fix_gamma=False, + momentum=bn_mom, + eps=2e-5, + name=name + '_bn2') if use_se: - #se begin - body = mx.sym.Pooling(data=bn2, global_pool=True, kernel=(7, 7), pool_type='avg', name=name+'_se_pool1') - body = Conv(data=body, num_filter=num_filter//16, kernel=(1,1), stride=(1,1), pad=(0,0), - name=name+"_se_conv1", workspace=workspace) - body = Act(data=body, act_type=act_type, name=name+'_se_relu1') - body = Conv(data=body, num_filter=num_filter, kernel=(1,1), stride=(1,1), pad=(0,0), - name=name+"_se_conv2", workspace=workspace) - body = mx.symbol.Activation(data=body, act_type='sigmoid', name=name+"_se_sigmoid") - bn2 = mx.symbol.broadcast_mul(bn2, body) - #se end + #se begin + body = mx.sym.Pooling(data=bn2, + global_pool=True, + kernel=(7, 7), + pool_type='avg', + name=name + '_se_pool1') + body = Conv(data=body, + num_filter=num_filter // 16, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv1", + workspace=workspace) + body = Act(data=body, act_type=act_type, name=name + '_se_relu1') + body = Conv(data=body, + num_filter=num_filter, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv2", + workspace=workspace) + body = mx.symbol.Activation(data=body, + act_type='sigmoid', + name=name + "_se_sigmoid") + bn2 = mx.symbol.broadcast_mul(bn2, body) + #se end if dim_match: shortcut = data else: - conv1sc = Conv(data=data, num_filter=num_filter, kernel=(1,1), stride=stride, no_bias=True, - workspace=workspace, name=name+'_conv1sc') - shortcut = mx.sym.BatchNorm(data=conv1sc, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_sc') + conv1sc = Conv(data=data, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + no_bias=True, + workspace=workspace, + name=name + '_conv1sc') + shortcut = mx.sym.BatchNorm(data=conv1sc, + fix_gamma=False, + momentum=bn_mom, + eps=2e-5, + name=name + '_sc') if memonger: shortcut._set_attr(mirror_stage='True') - return Act(data=bn2 + shortcut, act_type=act_type, name=name + '_relu3') + return Act(data=bn2 + shortcut, + act_type=act_type, + name=name + '_relu3') -def residual_unit_v1_L(data, num_filter, stride, dim_match, name, bottle_neck, **kwargs): + +def residual_unit_v1_L(data, num_filter, stride, dim_match, name, bottle_neck, + **kwargs): """Return ResNet Unit symbol for building ResNet Parameters ---------- @@ -162,70 +270,176 @@ def residual_unit_v1_L(data, num_filter, stride, dim_match, name, bottle_neck, * act_type = kwargs.get('version_act', 'prelu') #print('in unit1') if bottle_neck: - conv1 = Conv(data=data, num_filter=int(num_filter*0.25), kernel=(1,1), stride=(1,1), pad=(0,0), - no_bias=True, workspace=workspace, name=name + '_conv1') - bn1 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn1') + conv1 = Conv(data=data, + num_filter=int(num_filter * 0.25), + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + no_bias=True, + workspace=workspace, + name=name + '_conv1') + bn1 = mx.sym.BatchNorm(data=conv1, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn1') act1 = Act(data=bn1, act_type=act_type, name=name + '_relu1') - conv2 = Conv(data=act1, num_filter=int(num_filter*0.25), kernel=(3,3), stride=(1,1), pad=(1,1), - no_bias=True, workspace=workspace, name=name + '_conv2') - bn2 = mx.sym.BatchNorm(data=conv2, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn2') + conv2 = Conv(data=act1, + num_filter=int(num_filter * 0.25), + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv2') + bn2 = mx.sym.BatchNorm(data=conv2, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn2') act2 = Act(data=bn2, act_type=act_type, name=name + '_relu2') - conv3 = Conv(data=act2, num_filter=num_filter, kernel=(1,1), stride=stride, pad=(0,0), no_bias=True, - workspace=workspace, name=name + '_conv3') - bn3 = mx.sym.BatchNorm(data=conv3, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn3') + conv3 = Conv(data=act2, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + pad=(0, 0), + no_bias=True, + workspace=workspace, + name=name + '_conv3') + bn3 = mx.sym.BatchNorm(data=conv3, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn3') if use_se: - #se begin - body = mx.sym.Pooling(data=bn3, global_pool=True, kernel=(7, 7), pool_type='avg', name=name+'_se_pool1') - body = Conv(data=body, num_filter=num_filter//16, kernel=(1,1), stride=(1,1), pad=(0,0), - name=name+"_se_conv1", workspace=workspace) - body = Act(data=body, act_type=act_type, name=name+'_se_relu1') - body = Conv(data=body, num_filter=num_filter, kernel=(1,1), stride=(1,1), pad=(0,0), - name=name+"_se_conv2", workspace=workspace) - body = mx.symbol.Activation(data=body, act_type='sigmoid', name=name+"_se_sigmoid") - bn3 = mx.symbol.broadcast_mul(bn3, body) - #se end + #se begin + body = mx.sym.Pooling(data=bn3, + global_pool=True, + kernel=(7, 7), + pool_type='avg', + name=name + '_se_pool1') + body = Conv(data=body, + num_filter=num_filter // 16, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv1", + workspace=workspace) + body = Act(data=body, act_type=act_type, name=name + '_se_relu1') + body = Conv(data=body, + num_filter=num_filter, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv2", + workspace=workspace) + body = mx.symbol.Activation(data=body, + act_type='sigmoid', + name=name + "_se_sigmoid") + bn3 = mx.symbol.broadcast_mul(bn3, body) + #se end if dim_match: shortcut = data else: - conv1sc = Conv(data=data, num_filter=num_filter, kernel=(1,1), stride=stride, no_bias=True, - workspace=workspace, name=name+'_conv1sc') - shortcut = mx.sym.BatchNorm(data=conv1sc, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_sc') + conv1sc = Conv(data=data, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + no_bias=True, + workspace=workspace, + name=name + '_conv1sc') + shortcut = mx.sym.BatchNorm(data=conv1sc, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_sc') if memonger: shortcut._set_attr(mirror_stage='True') - return Act(data=bn3 + shortcut, act_type=act_type, name=name + '_relu3') + return Act(data=bn3 + shortcut, + act_type=act_type, + name=name + '_relu3') else: - conv1 = Conv(data=data, num_filter=num_filter, kernel=(3,3), stride=(1,1), pad=(1,1), - no_bias=True, workspace=workspace, name=name + '_conv1') - bn1 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_bn1') + conv1 = Conv(data=data, + num_filter=num_filter, + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv1') + bn1 = mx.sym.BatchNorm(data=conv1, + fix_gamma=False, + momentum=bn_mom, + eps=2e-5, + name=name + '_bn1') act1 = Act(data=bn1, act_type=act_type, name=name + '_relu1') - conv2 = Conv(data=act1, num_filter=num_filter, kernel=(3,3), stride=stride, pad=(1,1), - no_bias=True, workspace=workspace, name=name + '_conv2') - bn2 = mx.sym.BatchNorm(data=conv2, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_bn2') + conv2 = Conv(data=act1, + num_filter=num_filter, + kernel=(3, 3), + stride=stride, + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv2') + bn2 = mx.sym.BatchNorm(data=conv2, + fix_gamma=False, + momentum=bn_mom, + eps=2e-5, + name=name + '_bn2') if use_se: - #se begin - body = mx.sym.Pooling(data=bn2, global_pool=True, kernel=(7, 7), pool_type='avg', name=name+'_se_pool1') - body = Conv(data=body, num_filter=num_filter//16, kernel=(1,1), stride=(1,1), pad=(0,0), - name=name+"_se_conv1", workspace=workspace) - body = Act(data=body, act_type=act_type, name=name+'_se_relu1') - body = Conv(data=body, num_filter=num_filter, kernel=(1,1), stride=(1,1), pad=(0,0), - name=name+"_se_conv2", workspace=workspace) - body = mx.symbol.Activation(data=body, act_type='sigmoid', name=name+"_se_sigmoid") - bn2 = mx.symbol.broadcast_mul(bn2, body) - #se end + #se begin + body = mx.sym.Pooling(data=bn2, + global_pool=True, + kernel=(7, 7), + pool_type='avg', + name=name + '_se_pool1') + body = Conv(data=body, + num_filter=num_filter // 16, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv1", + workspace=workspace) + body = Act(data=body, act_type=act_type, name=name + '_se_relu1') + body = Conv(data=body, + num_filter=num_filter, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv2", + workspace=workspace) + body = mx.symbol.Activation(data=body, + act_type='sigmoid', + name=name + "_se_sigmoid") + bn2 = mx.symbol.broadcast_mul(bn2, body) + #se end if dim_match: shortcut = data else: - conv1sc = Conv(data=data, num_filter=num_filter, kernel=(1,1), stride=stride, no_bias=True, - workspace=workspace, name=name+'_conv1sc') - shortcut = mx.sym.BatchNorm(data=conv1sc, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_sc') + conv1sc = Conv(data=data, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + no_bias=True, + workspace=workspace, + name=name + '_conv1sc') + shortcut = mx.sym.BatchNorm(data=conv1sc, + fix_gamma=False, + momentum=bn_mom, + eps=2e-5, + name=name + '_sc') if memonger: shortcut._set_attr(mirror_stage='True') - return Act(data=bn2 + shortcut, act_type=act_type, name=name + '_relu3') + return Act(data=bn2 + shortcut, + act_type=act_type, + name=name + '_relu3') -def residual_unit_v2(data, num_filter, stride, dim_match, name, bottle_neck, **kwargs): + +def residual_unit_v2(data, num_filter, stride, dim_match, name, bottle_neck, + **kwargs): """Return ResNet Unit symbol for building ResNet Parameters ---------- @@ -252,66 +466,159 @@ def residual_unit_v2(data, num_filter, stride, dim_match, name, bottle_neck, **k #print('in unit2') if bottle_neck: # the same as https://github.com/facebook/fb.resnet.torch#notes, a bit difference with origin paper - bn1 = mx.sym.BatchNorm(data=data, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn1') + bn1 = mx.sym.BatchNorm(data=data, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn1') act1 = Act(data=bn1, act_type=act_type, name=name + '_relu1') - conv1 = Conv(data=act1, num_filter=int(num_filter*0.25), kernel=(1,1), stride=(1,1), pad=(0,0), - no_bias=True, workspace=workspace, name=name + '_conv1') - bn2 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn2') + conv1 = Conv(data=act1, + num_filter=int(num_filter * 0.25), + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + no_bias=True, + workspace=workspace, + name=name + '_conv1') + bn2 = mx.sym.BatchNorm(data=conv1, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn2') act2 = Act(data=bn2, act_type=act_type, name=name + '_relu2') - conv2 = Conv(data=act2, num_filter=int(num_filter*0.25), kernel=(3,3), stride=stride, pad=(1,1), - no_bias=True, workspace=workspace, name=name + '_conv2') - bn3 = mx.sym.BatchNorm(data=conv2, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn3') + conv2 = Conv(data=act2, + num_filter=int(num_filter * 0.25), + kernel=(3, 3), + stride=stride, + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv2') + bn3 = mx.sym.BatchNorm(data=conv2, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn3') act3 = Act(data=bn3, act_type=act_type, name=name + '_relu3') - conv3 = Conv(data=act3, num_filter=num_filter, kernel=(1,1), stride=(1,1), pad=(0,0), no_bias=True, - workspace=workspace, name=name + '_conv3') + conv3 = Conv(data=act3, + num_filter=num_filter, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + no_bias=True, + workspace=workspace, + name=name + '_conv3') if use_se: - #se begin - body = mx.sym.Pooling(data=conv3, global_pool=True, kernel=(7, 7), pool_type='avg', name=name+'_se_pool1') - body = Conv(data=body, num_filter=num_filter//16, kernel=(1,1), stride=(1,1), pad=(0,0), - name=name+"_se_conv1", workspace=workspace) - body = Act(data=body, act_type=act_type, name=name+'_se_relu1') - body = Conv(data=body, num_filter=num_filter, kernel=(1,1), stride=(1,1), pad=(0,0), - name=name+"_se_conv2", workspace=workspace) - body = mx.symbol.Activation(data=body, act_type='sigmoid', name=name+"_se_sigmoid") - conv3 = mx.symbol.broadcast_mul(conv3, body) + #se begin + body = mx.sym.Pooling(data=conv3, + global_pool=True, + kernel=(7, 7), + pool_type='avg', + name=name + '_se_pool1') + body = Conv(data=body, + num_filter=num_filter // 16, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv1", + workspace=workspace) + body = Act(data=body, act_type=act_type, name=name + '_se_relu1') + body = Conv(data=body, + num_filter=num_filter, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv2", + workspace=workspace) + body = mx.symbol.Activation(data=body, + act_type='sigmoid', + name=name + "_se_sigmoid") + conv3 = mx.symbol.broadcast_mul(conv3, body) if dim_match: shortcut = data else: - shortcut = Conv(data=act1, num_filter=num_filter, kernel=(1,1), stride=stride, no_bias=True, - workspace=workspace, name=name+'_sc') + shortcut = Conv(data=act1, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + no_bias=True, + workspace=workspace, + name=name + '_sc') if memonger: shortcut._set_attr(mirror_stage='True') return conv3 + shortcut else: - bn1 = mx.sym.BatchNorm(data=data, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_bn1') + bn1 = mx.sym.BatchNorm(data=data, + fix_gamma=False, + momentum=bn_mom, + eps=2e-5, + name=name + '_bn1') act1 = Act(data=bn1, act_type=act_type, name=name + '_relu1') - conv1 = Conv(data=act1, num_filter=num_filter, kernel=(3,3), stride=stride, pad=(1,1), - no_bias=True, workspace=workspace, name=name + '_conv1') - bn2 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_bn2') + conv1 = Conv(data=act1, + num_filter=num_filter, + kernel=(3, 3), + stride=stride, + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv1') + bn2 = mx.sym.BatchNorm(data=conv1, + fix_gamma=False, + momentum=bn_mom, + eps=2e-5, + name=name + '_bn2') act2 = Act(data=bn2, act_type=act_type, name=name + '_relu2') - conv2 = Conv(data=act2, num_filter=num_filter, kernel=(3,3), stride=(1,1), pad=(1,1), - no_bias=True, workspace=workspace, name=name + '_conv2') + conv2 = Conv(data=act2, + num_filter=num_filter, + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv2') if use_se: - #se begin - body = mx.sym.Pooling(data=conv2, global_pool=True, kernel=(7, 7), pool_type='avg', name=name+'_se_pool1') - body = Conv(data=body, num_filter=num_filter//16, kernel=(1,1), stride=(1,1), pad=(0,0), - name=name+"_se_conv1", workspace=workspace) - body = Act(data=body, act_type=act_type, name=name+'_se_relu1') - body = Conv(data=body, num_filter=num_filter, kernel=(1,1), stride=(1,1), pad=(0,0), - name=name+"_se_conv2", workspace=workspace) - body = mx.symbol.Activation(data=body, act_type='sigmoid', name=name+"_se_sigmoid") - conv2 = mx.symbol.broadcast_mul(conv2, body) + #se begin + body = mx.sym.Pooling(data=conv2, + global_pool=True, + kernel=(7, 7), + pool_type='avg', + name=name + '_se_pool1') + body = Conv(data=body, + num_filter=num_filter // 16, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv1", + workspace=workspace) + body = Act(data=body, act_type=act_type, name=name + '_se_relu1') + body = Conv(data=body, + num_filter=num_filter, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv2", + workspace=workspace) + body = mx.symbol.Activation(data=body, + act_type='sigmoid', + name=name + "_se_sigmoid") + conv2 = mx.symbol.broadcast_mul(conv2, body) if dim_match: shortcut = data else: - shortcut = Conv(data=act1, num_filter=num_filter, kernel=(1,1), stride=stride, no_bias=True, - workspace=workspace, name=name+'_sc') + shortcut = Conv(data=act1, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + no_bias=True, + workspace=workspace, + name=name + '_sc') if memonger: shortcut._set_attr(mirror_stage='True') return conv2 + shortcut -def residual_unit_v3(data, num_filter, stride, dim_match, name, bottle_neck, **kwargs): - + +def residual_unit_v3(data, num_filter, stride, dim_match, name, bottle_neck, + **kwargs): """Return ResNet Unit symbol for building ResNet Parameters ---------- @@ -337,73 +644,182 @@ def residual_unit_v3(data, num_filter, stride, dim_match, name, bottle_neck, **k act_type = kwargs.get('version_act', 'prelu') #print('in unit3') if bottle_neck: - bn1 = mx.sym.BatchNorm(data=data, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn1') - conv1 = Conv(data=bn1, num_filter=int(num_filter*0.25), kernel=(1,1), stride=(1,1), pad=(0,0), - no_bias=True, workspace=workspace, name=name + '_conv1') - bn2 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn2') + bn1 = mx.sym.BatchNorm(data=data, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn1') + conv1 = Conv(data=bn1, + num_filter=int(num_filter * 0.25), + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + no_bias=True, + workspace=workspace, + name=name + '_conv1') + bn2 = mx.sym.BatchNorm(data=conv1, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn2') act1 = Act(data=bn2, act_type=act_type, name=name + '_relu1') - conv2 = Conv(data=act1, num_filter=int(num_filter*0.25), kernel=(3,3), stride=(1,1), pad=(1,1), - no_bias=True, workspace=workspace, name=name + '_conv2') - bn3 = mx.sym.BatchNorm(data=conv2, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn3') + conv2 = Conv(data=act1, + num_filter=int(num_filter * 0.25), + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv2') + bn3 = mx.sym.BatchNorm(data=conv2, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn3') act2 = Act(data=bn3, act_type=act_type, name=name + '_relu2') - conv3 = Conv(data=act2, num_filter=num_filter, kernel=(1,1), stride=stride, pad=(0,0), no_bias=True, - workspace=workspace, name=name + '_conv3') - bn4 = mx.sym.BatchNorm(data=conv3, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn4') + conv3 = Conv(data=act2, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + pad=(0, 0), + no_bias=True, + workspace=workspace, + name=name + '_conv3') + bn4 = mx.sym.BatchNorm(data=conv3, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn4') if use_se: - #se begin - body = mx.sym.Pooling(data=bn4, global_pool=True, kernel=(7, 7), pool_type='avg', name=name+'_se_pool1') - body = Conv(data=body, num_filter=num_filter//16, kernel=(1,1), stride=(1,1), pad=(0,0), - name=name+"_se_conv1", workspace=workspace) - body = Act(data=body, act_type=act_type, name=name+'_se_relu1') - body = Conv(data=body, num_filter=num_filter, kernel=(1,1), stride=(1,1), pad=(0,0), - name=name+"_se_conv2", workspace=workspace) - body = mx.symbol.Activation(data=body, act_type='sigmoid', name=name+"_se_sigmoid") - bn4 = mx.symbol.broadcast_mul(bn4, body) - #se end + #se begin + body = mx.sym.Pooling(data=bn4, + global_pool=True, + kernel=(7, 7), + pool_type='avg', + name=name + '_se_pool1') + body = Conv(data=body, + num_filter=num_filter // 16, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv1", + workspace=workspace) + body = Act(data=body, act_type=act_type, name=name + '_se_relu1') + body = Conv(data=body, + num_filter=num_filter, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv2", + workspace=workspace) + body = mx.symbol.Activation(data=body, + act_type='sigmoid', + name=name + "_se_sigmoid") + bn4 = mx.symbol.broadcast_mul(bn4, body) + #se end if dim_match: shortcut = data else: - conv1sc = Conv(data=data, num_filter=num_filter, kernel=(1,1), stride=stride, no_bias=True, - workspace=workspace, name=name+'_conv1sc') - shortcut = mx.sym.BatchNorm(data=conv1sc, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_sc') + conv1sc = Conv(data=data, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + no_bias=True, + workspace=workspace, + name=name + '_conv1sc') + shortcut = mx.sym.BatchNorm(data=conv1sc, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_sc') if memonger: shortcut._set_attr(mirror_stage='True') return bn4 + shortcut else: - bn1 = mx.sym.BatchNorm(data=data, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn1') - conv1 = Conv(data=bn1, num_filter=num_filter, kernel=(3,3), stride=(1,1), pad=(1,1), - no_bias=True, workspace=workspace, name=name + '_conv1') - bn2 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn2') + bn1 = mx.sym.BatchNorm(data=data, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn1') + conv1 = Conv(data=bn1, + num_filter=num_filter, + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv1') + bn2 = mx.sym.BatchNorm(data=conv1, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn2') act1 = Act(data=bn2, act_type=act_type, name=name + '_relu1') - conv2 = Conv(data=act1, num_filter=num_filter, kernel=(3,3), stride=stride, pad=(1,1), - no_bias=True, workspace=workspace, name=name + '_conv2') - bn3 = mx.sym.BatchNorm(data=conv2, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn3') + conv2 = Conv(data=act1, + num_filter=num_filter, + kernel=(3, 3), + stride=stride, + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv2') + bn3 = mx.sym.BatchNorm(data=conv2, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn3') if use_se: - #se begin - body = mx.sym.Pooling(data=bn3, global_pool=True, kernel=(7, 7), pool_type='avg', name=name+'_se_pool1') - body = Conv(data=body, num_filter=num_filter//16, kernel=(1,1), stride=(1,1), pad=(0,0), - name=name+"_se_conv1", workspace=workspace) - body = Act(data=body, act_type=act_type, name=name+'_se_relu1') - body = Conv(data=body, num_filter=num_filter, kernel=(1,1), stride=(1,1), pad=(0,0), - name=name+"_se_conv2", workspace=workspace) - body = mx.symbol.Activation(data=body, act_type='sigmoid', name=name+"_se_sigmoid") - bn3 = mx.symbol.broadcast_mul(bn3, body) - #se end + #se begin + body = mx.sym.Pooling(data=bn3, + global_pool=True, + kernel=(7, 7), + pool_type='avg', + name=name + '_se_pool1') + body = Conv(data=body, + num_filter=num_filter // 16, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv1", + workspace=workspace) + body = Act(data=body, act_type=act_type, name=name + '_se_relu1') + body = Conv(data=body, + num_filter=num_filter, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv2", + workspace=workspace) + body = mx.symbol.Activation(data=body, + act_type='sigmoid', + name=name + "_se_sigmoid") + bn3 = mx.symbol.broadcast_mul(bn3, body) + #se end if dim_match: shortcut = data else: - conv1sc = Conv(data=data, num_filter=num_filter, kernel=(1,1), stride=stride, no_bias=True, - workspace=workspace, name=name+'_conv1sc') - shortcut = mx.sym.BatchNorm(data=conv1sc, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_sc') + conv1sc = Conv(data=data, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + no_bias=True, + workspace=workspace, + name=name + '_conv1sc') + shortcut = mx.sym.BatchNorm(data=conv1sc, + fix_gamma=False, + momentum=bn_mom, + eps=2e-5, + name=name + '_sc') if memonger: shortcut._set_attr(mirror_stage='True') return bn3 + shortcut -def residual_unit_v3_x(data, num_filter, stride, dim_match, name, bottle_neck, **kwargs): - + +def residual_unit_v3_x(data, num_filter, stride, dim_match, name, bottle_neck, + **kwargs): """Return ResNeXt Unit symbol for building ResNeXt Parameters ---------- @@ -422,7 +838,7 @@ def residual_unit_v3_x(data, num_filter, stride, dim_match, name, bottle_neck, * workspace : int Workspace used in convolution operator """ - assert(bottle_neck) + assert (bottle_neck) use_se = kwargs.get('version_se', 1) bn_mom = kwargs.get('bn_mom', 0.9) workspace = kwargs.get('workspace', 256) @@ -430,56 +846,124 @@ def residual_unit_v3_x(data, num_filter, stride, dim_match, name, bottle_neck, * act_type = kwargs.get('version_act', 'prelu') num_group = 32 #print('in unit3') - bn1 = mx.sym.BatchNorm(data=data, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn1') - conv1 = Conv(data=bn1, num_group=num_group, num_filter=int(num_filter*0.5), kernel=(1,1), stride=(1,1), pad=(0,0), - no_bias=True, workspace=workspace, name=name + '_conv1') - bn2 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn2') + bn1 = mx.sym.BatchNorm(data=data, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn1') + conv1 = Conv(data=bn1, + num_group=num_group, + num_filter=int(num_filter * 0.5), + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + no_bias=True, + workspace=workspace, + name=name + '_conv1') + bn2 = mx.sym.BatchNorm(data=conv1, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn2') act1 = Act(data=bn2, act_type=act_type, name=name + '_relu1') - conv2 = Conv(data=act1, num_group=num_group, num_filter=int(num_filter*0.5), kernel=(3,3), stride=(1,1), pad=(1,1), - no_bias=True, workspace=workspace, name=name + '_conv2') - bn3 = mx.sym.BatchNorm(data=conv2, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn3') + conv2 = Conv(data=act1, + num_group=num_group, + num_filter=int(num_filter * 0.5), + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv2') + bn3 = mx.sym.BatchNorm(data=conv2, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn3') act2 = Act(data=bn3, act_type=act_type, name=name + '_relu2') - conv3 = Conv(data=act2, num_filter=num_filter, kernel=(1,1), stride=stride, pad=(0,0), no_bias=True, - workspace=workspace, name=name + '_conv3') - bn4 = mx.sym.BatchNorm(data=conv3, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn4') + conv3 = Conv(data=act2, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + pad=(0, 0), + no_bias=True, + workspace=workspace, + name=name + '_conv3') + bn4 = mx.sym.BatchNorm(data=conv3, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn4') if use_se: - #se begin - body = mx.sym.Pooling(data=bn4, global_pool=True, kernel=(7, 7), pool_type='avg', name=name+'_se_pool1') - body = Conv(data=body, num_filter=num_filter//16, kernel=(1,1), stride=(1,1), pad=(0,0), - name=name+"_se_conv1", workspace=workspace) - body = Act(data=body, act_type=act_type, name=name+'_se_relu1') - body = Conv(data=body, num_filter=num_filter, kernel=(1,1), stride=(1,1), pad=(0,0), - name=name+"_se_conv2", workspace=workspace) - body = mx.symbol.Activation(data=body, act_type='sigmoid', name=name+"_se_sigmoid") - bn4 = mx.symbol.broadcast_mul(bn4, body) - #se end + #se begin + body = mx.sym.Pooling(data=bn4, + global_pool=True, + kernel=(7, 7), + pool_type='avg', + name=name + '_se_pool1') + body = Conv(data=body, + num_filter=num_filter // 16, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv1", + workspace=workspace) + body = Act(data=body, act_type=act_type, name=name + '_se_relu1') + body = Conv(data=body, + num_filter=num_filter, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv2", + workspace=workspace) + body = mx.symbol.Activation(data=body, + act_type='sigmoid', + name=name + "_se_sigmoid") + bn4 = mx.symbol.broadcast_mul(bn4, body) + #se end if dim_match: shortcut = data else: - conv1sc = Conv(data=data, num_filter=num_filter, kernel=(1,1), stride=stride, no_bias=True, - workspace=workspace, name=name+'_conv1sc') - shortcut = mx.sym.BatchNorm(data=conv1sc, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_sc') + conv1sc = Conv(data=data, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + no_bias=True, + workspace=workspace, + name=name + '_conv1sc') + shortcut = mx.sym.BatchNorm(data=conv1sc, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_sc') if memonger: shortcut._set_attr(mirror_stage='True') return bn4 + shortcut -def residual_unit(data, num_filter, stride, dim_match, name, bottle_neck, **kwargs): - uv = kwargs.get('version_unit', 3) - version_input = kwargs.get('version_input', 1) - if uv==1: - if version_input==0: - return residual_unit_v1(data, num_filter, stride, dim_match, name, bottle_neck, **kwargs) +def residual_unit(data, num_filter, stride, dim_match, name, bottle_neck, + **kwargs): + uv = kwargs.get('version_unit', 3) + version_input = kwargs.get('version_input', 1) + if uv == 1: + if version_input == 0: + return residual_unit_v1(data, num_filter, stride, dim_match, name, + bottle_neck, **kwargs) + else: + return residual_unit_v1_L(data, num_filter, stride, dim_match, + name, bottle_neck, **kwargs) + elif uv == 2: + return residual_unit_v2(data, num_filter, stride, dim_match, name, + bottle_neck, **kwargs) + elif uv == 4: + return residual_unit_v4(data, num_filter, stride, dim_match, name, + bottle_neck, **kwargs) else: - return residual_unit_v1_L(data, num_filter, stride, dim_match, name, bottle_neck, **kwargs) - elif uv==2: - return residual_unit_v2(data, num_filter, stride, dim_match, name, bottle_neck, **kwargs) - elif uv==4: - return residual_unit_v4(data, num_filter, stride, dim_match, name, bottle_neck, **kwargs) - else: - return residual_unit_v3(data, num_filter, stride, dim_match, name, bottle_neck, **kwargs) + return residual_unit_v3(data, num_filter, stride, dim_match, name, + bottle_neck, **kwargs) + def resnet(units, num_stages, filter_list, num_classes, bottle_neck, **kwargs): bn_mom = kwargs.get('bn_mom', 0.9) @@ -502,57 +986,100 @@ def resnet(units, num_stages, filter_list, num_classes, bottle_neck, **kwargs): """ version_se = kwargs.get('version_se', 0) version_input = kwargs.get('version_input', 1) - assert version_input>=0 + assert version_input >= 0 version_output = kwargs.get('version_output', 'GAP') fc_type = version_output version_unit = kwargs.get('version_unit', 3) act_type = kwargs.get('version_act', 'prelu') print(version_se, version_input, version_output, version_unit, act_type) num_unit = len(units) - assert(num_unit == num_stages) + assert (num_unit == num_stages) data = mx.sym.Variable(name='data') - if version_input==0: - #data = mx.sym.BatchNorm(data=data, fix_gamma=True, eps=2e-5, momentum=bn_mom, name='bn_data') - data = mx.sym.identity(data=data, name='id') - data = data-127.5 - data = data*0.0078125 - body = Conv(data=data, num_filter=filter_list[0], kernel=(7, 7), stride=(2,2), pad=(3, 3), - no_bias=True, name="conv0", workspace=workspace) - body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn0') - body = Act(data=body, act_type=act_type, name='relu0') - #body = mx.sym.Pooling(data=body, kernel=(3, 3), stride=(2,2), pad=(1,1), pool_type='max') - elif version_input==2: - data = mx.sym.BatchNorm(data=data, fix_gamma=True, eps=2e-5, momentum=bn_mom, name='bn_data') - body = Conv(data=data, num_filter=filter_list[0], kernel=(3,3), stride=(1,1), pad=(1,1), - no_bias=True, name="conv0", workspace=workspace) - body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn0') - body = Act(data=body, act_type=act_type, name='relu0') + if version_input == 0: + #data = mx.sym.BatchNorm(data=data, fix_gamma=True, eps=2e-5, momentum=bn_mom, name='bn_data') + data = mx.sym.identity(data=data, name='id') + data = data - 127.5 + data = data * 0.0078125 + body = Conv(data=data, + num_filter=filter_list[0], + kernel=(7, 7), + stride=(2, 2), + pad=(3, 3), + no_bias=True, + name="conv0", + workspace=workspace) + body = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name='bn0') + body = Act(data=body, act_type=act_type, name='relu0') + #body = mx.sym.Pooling(data=body, kernel=(3, 3), stride=(2,2), pad=(1,1), pool_type='max') + elif version_input == 2: + data = mx.sym.BatchNorm(data=data, + fix_gamma=True, + eps=2e-5, + momentum=bn_mom, + name='bn_data') + body = Conv(data=data, + num_filter=filter_list[0], + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + name="conv0", + workspace=workspace) + body = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name='bn0') + body = Act(data=body, act_type=act_type, name='relu0') else: - data = mx.sym.identity(data=data, name='id') - data = data-127.5 - data = data*0.0078125 - body = data - body = Conv(data=body, num_filter=filter_list[0], kernel=(3,3), stride=(1,1), pad=(1, 1), - no_bias=True, name="conv0", workspace=workspace) - body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn0') - body = Act(data=body, act_type=act_type, name='relu0') + data = mx.sym.identity(data=data, name='id') + data = data - 127.5 + data = data * 0.0078125 + body = data + body = Conv(data=body, + num_filter=filter_list[0], + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + name="conv0", + workspace=workspace) + body = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name='bn0') + body = Act(data=body, act_type=act_type, name='relu0') for i in range(num_stages): - #if version_input==0: - # body = residual_unit(body, filter_list[i+1], (1 if i==0 else 2, 1 if i==0 else 2), False, - # name='stage%d_unit%d' % (i + 1, 1), bottle_neck=bottle_neck, **kwargs) - #else: - # body = residual_unit(body, filter_list[i+1], (2, 2), False, - # name='stage%d_unit%d' % (i + 1, 1), bottle_neck=bottle_neck, **kwargs) - body = residual_unit(body, filter_list[i+1], (2, 2), False, - name='stage%d_unit%d' % (i + 1, 1), bottle_neck=bottle_neck, **kwargs) - for j in range(units[i]-1): - body = residual_unit(body, filter_list[i+1], (1,1), True, name='stage%d_unit%d' % (i+1, j+2), - bottle_neck=bottle_neck, **kwargs) + #if version_input==0: + # body = residual_unit(body, filter_list[i+1], (1 if i==0 else 2, 1 if i==0 else 2), False, + # name='stage%d_unit%d' % (i + 1, 1), bottle_neck=bottle_neck, **kwargs) + #else: + # body = residual_unit(body, filter_list[i+1], (2, 2), False, + # name='stage%d_unit%d' % (i + 1, 1), bottle_neck=bottle_neck, **kwargs) + body = residual_unit(body, + filter_list[i + 1], (2, 2), + False, + name='stage%d_unit%d' % (i + 1, 1), + bottle_neck=bottle_neck, + **kwargs) + for j in range(units[i] - 1): + body = residual_unit(body, + filter_list[i + 1], (1, 1), + True, + name='stage%d_unit%d' % (i + 1, j + 2), + bottle_neck=bottle_neck, + **kwargs) fc1 = symbol_utils.get_fc1(body, num_classes, fc_type) return fc1 + def get_symbol(num_classes, num_layers, **kwargs): """ Adapted from https://github.com/tornadomeet/ResNet/blob/master/train_resnet.py @@ -590,12 +1117,13 @@ def get_symbol(num_classes, num_layers, **kwargs): elif num_layers == 269: units = [3, 30, 48, 8] else: - raise ValueError("no experiments done on num_layers {}, you can do it yourself".format(num_layers)) + raise ValueError( + "no experiments done on num_layers {}, you can do it yourself". + format(num_layers)) - return resnet(units = units, - num_stages = num_stages, - filter_list = filter_list, - num_classes = num_classes, - bottle_neck = bottle_neck, + return resnet(units=units, + num_stages=num_stages, + filter_list=filter_list, + num_classes=num_classes, + bottle_neck=bottle_neck, **kwargs) - diff --git a/gender-age/helper.py b/gender-age/helper.py index b82c4b7..38f2c9c 100644 --- a/gender-age/helper.py +++ b/gender-age/helper.py @@ -61,11 +61,13 @@ def nms(boxes, overlap_threshold, mode='Union'): overlap = inter / (area[i] + area[idxs[:last]] - inter) # delete all indexes from the index list that have - idxs = np.delete(idxs, np.concatenate(([last], - np.where(overlap > overlap_threshold)[0]))) + idxs = np.delete( + idxs, + np.concatenate(([last], np.where(overlap > overlap_threshold)[0]))) return pick + def adjust_input(in_data): """ adjust the input from (h, w, c) to ( 1, c, h, w) for network input @@ -84,13 +86,14 @@ def adjust_input(in_data): else: out_data = in_data - out_data = out_data.transpose((2,0,1)) + out_data = out_data.transpose((2, 0, 1)) out_data = np.expand_dims(out_data, 0) - out_data = (out_data - 127.5)*0.0078125 + out_data = (out_data - 127.5) * 0.0078125 return out_data + def generate_bbox(map, reg, scale, threshold): - """ + """ generate bbox from feature map Parameters: ---------- @@ -106,27 +109,27 @@ def generate_bbox(map, reg, scale, threshold): ------- bbox array """ - stride = 2 - cellsize = 12 + stride = 2 + cellsize = 12 - t_index = np.where(map>threshold) + t_index = np.where(map > threshold) - # find nothing - if t_index[0].size == 0: - return np.array([]) + # find nothing + if t_index[0].size == 0: + return np.array([]) - dx1, dy1, dx2, dy2 = [reg[0, i, t_index[0], t_index[1]] for i in range(4)] + dx1, dy1, dx2, dy2 = [reg[0, i, t_index[0], t_index[1]] for i in range(4)] - reg = np.array([dx1, dy1, dx2, dy2]) - score = map[t_index[0], t_index[1]] - boundingbox = np.vstack([np.round((stride*t_index[1]+1)/scale), - np.round((stride*t_index[0]+1)/scale), - np.round((stride*t_index[1]+1+cellsize)/scale), - np.round((stride*t_index[0]+1+cellsize)/scale), - score, - reg]) + reg = np.array([dx1, dy1, dx2, dy2]) + score = map[t_index[0], t_index[1]] + boundingbox = np.vstack([ + np.round((stride * t_index[1] + 1) / scale), + np.round((stride * t_index[0] + 1) / scale), + np.round((stride * t_index[1] + 1 + cellsize) / scale), + np.round((stride * t_index[0] + 1 + cellsize) / scale), score, reg + ]) - return boundingbox.T + return boundingbox.T def detect_first_stage(img, net, scale, threshold): @@ -148,21 +151,22 @@ def detect_first_stage(img, net, scale, threshold): height, width, _ = img.shape hs = int(math.ceil(height * scale)) ws = int(math.ceil(width * scale)) - - im_data = cv2.resize(img, (ws,hs)) - + + im_data = cv2.resize(img, (ws, hs)) + # adjust for the network input input_buf = adjust_input(im_data) output = net.predict(input_buf) - boxes = generate_bbox(output[1][0,1,:,:], output[0], scale, threshold) + boxes = generate_bbox(output[1][0, 1, :, :], output[0], scale, threshold) if boxes.size == 0: return None # nms - pick = nms(boxes[:,0:5], 0.5, mode='Union') + pick = nms(boxes[:, 0:5], 0.5, mode='Union') boxes = boxes[pick] return boxes -def detect_first_stage_warpper( args ): + +def detect_first_stage_warpper(args): return detect_first_stage(*args) diff --git a/gender-age/mtcnn_detector.py b/gender-age/mtcnn_detector.py index 2044f1d..1ce4146 100644 --- a/gender-age/mtcnn_detector.py +++ b/gender-age/mtcnn_detector.py @@ -9,6 +9,7 @@ from itertools import repeat from itertools import izip from helper import nms, adjust_input, generate_bbox, detect_first_stage_warpper + class MtcnnDetector(object): """ Joint Face Detection and Alignment using Multi-task Cascaded Convolutional Neural Networks @@ -17,11 +18,11 @@ class MtcnnDetector(object): """ def __init__(self, model_folder='.', - minsize = 20, - threshold = [0.6, 0.7, 0.8], - factor = 0.709, - num_worker = 1, - accurate_landmark = False, + minsize=20, + threshold=[0.6, 0.7, 0.8], + factor=0.709, + num_worker=1, + accurate_landmark=False, ctx=mx.cpu()): """ Initialize the detector @@ -46,9 +47,9 @@ class MtcnnDetector(object): self.accurate_landmark = accurate_landmark # load 4 models from folder - models = ['det1', 'det2', 'det3','det4'] - models = [ os.path.join(model_folder, f) for f in models] - + models = ['det1', 'det2', 'det3', 'det4'] + models = [os.path.join(model_folder, f) for f in models] + self.PNets = [] for i in range(num_worker): workner_net = mx.model.FeedForward.load(models[0], 1, ctx=ctx) @@ -60,11 +61,10 @@ class MtcnnDetector(object): self.ONet = mx.model.FeedForward.load(models[2], 1, ctx=ctx) self.LNet = mx.model.FeedForward.load(models[3], 1, ctx=ctx) - self.minsize = float(minsize) - self.factor = float(factor) + self.minsize = float(minsize) + self.factor = float(factor) self.threshold = threshold - def convert_to_square(self, bbox): """ convert bbox to square @@ -82,9 +82,9 @@ class MtcnnDetector(object): h = bbox[:, 3] - bbox[:, 1] + 1 w = bbox[:, 2] - bbox[:, 0] + 1 - max_side = np.maximum(h,w) - square_bbox[:, 0] = bbox[:, 0] + w*0.5 - max_side*0.5 - square_bbox[:, 1] = bbox[:, 1] + h*0.5 - max_side*0.5 + max_side = np.maximum(h, w) + square_bbox[:, 0] = bbox[:, 0] + w * 0.5 - max_side * 0.5 + square_bbox[:, 1] = bbox[:, 1] + h * 0.5 - max_side * 0.5 square_bbox[:, 2] = square_bbox[:, 0] + max_side - 1 square_bbox[:, 3] = square_bbox[:, 1] + max_side - 1 return square_bbox @@ -114,7 +114,6 @@ class MtcnnDetector(object): bbox[:, 0:4] = bbox[:, 0:4] + aug return bbox - def pad(self, bboxes, w, h): """ pad the the bboxes, alse restrict the size of it @@ -141,19 +140,21 @@ class MtcnnDetector(object): height and width of the bbox """ - tmpw, tmph = bboxes[:, 2] - bboxes[:, 0] + 1, bboxes[:, 3] - bboxes[:, 1] + 1 + tmpw, tmph = bboxes[:, 2] - bboxes[:, 0] + 1, bboxes[:, + 3] - bboxes[:, + 1] + 1 num_box = bboxes.shape[0] - dx , dy= np.zeros((num_box, )), np.zeros((num_box, )) - edx, edy = tmpw.copy()-1, tmph.copy()-1 + dx, dy = np.zeros((num_box, )), np.zeros((num_box, )) + edx, edy = tmpw.copy() - 1, tmph.copy() - 1 x, y, ex, ey = bboxes[:, 0], bboxes[:, 1], bboxes[:, 2], bboxes[:, 3] - tmp_index = np.where(ex > w-1) + tmp_index = np.where(ex > w - 1) edx[tmp_index] = tmpw[tmp_index] + w - 2 - ex[tmp_index] ex[tmp_index] = w - 1 - tmp_index = np.where(ey > h-1) + tmp_index = np.where(ey > h - 1) edy[tmp_index] = tmph[tmp_index] + h - 2 - ey[tmp_index] ey[tmp_index] = h - 1 @@ -168,7 +169,7 @@ class MtcnnDetector(object): return_list = [dy, edy, dx, edx, y, ey, x, ex, tmpw, tmph] return_list = [item.astype(np.int32) for item in return_list] - return return_list + return return_list def slice_index(self, number): """ @@ -182,53 +183,63 @@ class MtcnnDetector(object): """Yield successive n-sized chunks from l.""" for i in range(0, len(l), n): yield l[i:i + n] + num_list = range(number) return list(chunks(num_list, self.num_worker)) - + def detect_face_limited(self, img, det_type=2): height, width, _ = img.shape - if det_type>=2: - total_boxes = np.array( [ [0.0, 0.0, img.shape[1], img.shape[0], 0.9] ] ,dtype=np.float32) - num_box = total_boxes.shape[0] + if det_type >= 2: + total_boxes = np.array( + [[0.0, 0.0, img.shape[1], img.shape[0], 0.9]], + dtype=np.float32) + num_box = total_boxes.shape[0] - # pad the bbox - [dy, edy, dx, edx, y, ey, x, ex, tmpw, tmph] = self.pad(total_boxes, width, height) - # (3, 24, 24) is the input shape for RNet - input_buf = np.zeros((num_box, 3, 24, 24), dtype=np.float32) + # pad the bbox + [dy, edy, dx, edx, y, ey, x, ex, tmpw, + tmph] = self.pad(total_boxes, width, height) + # (3, 24, 24) is the input shape for RNet + input_buf = np.zeros((num_box, 3, 24, 24), dtype=np.float32) - for i in range(num_box): - tmp = np.zeros((tmph[i], tmpw[i], 3), dtype=np.uint8) - tmp[dy[i]:edy[i]+1, dx[i]:edx[i]+1, :] = img[y[i]:ey[i]+1, x[i]:ex[i]+1, :] - input_buf[i, :, :, :] = adjust_input(cv2.resize(tmp, (24, 24))) + for i in range(num_box): + tmp = np.zeros((tmph[i], tmpw[i], 3), dtype=np.uint8) + tmp[dy[i]:edy[i] + 1, + dx[i]:edx[i] + 1, :] = img[y[i]:ey[i] + 1, + x[i]:ex[i] + 1, :] + input_buf[i, :, :, :] = adjust_input(cv2.resize(tmp, (24, 24))) - output = self.RNet.predict(input_buf) + output = self.RNet.predict(input_buf) - # filter the total_boxes with threshold - passed = np.where(output[1][:, 1] > self.threshold[1]) - total_boxes = total_boxes[passed] + # filter the total_boxes with threshold + passed = np.where(output[1][:, 1] > self.threshold[1]) + total_boxes = total_boxes[passed] - if total_boxes.size == 0: - return None + if total_boxes.size == 0: + return None - total_boxes[:, 4] = output[1][passed, 1].reshape((-1,)) - reg = output[0][passed] + total_boxes[:, 4] = output[1][passed, 1].reshape((-1, )) + reg = output[0][passed] - # nms - pick = nms(total_boxes, 0.7, 'Union') - total_boxes = total_boxes[pick] - total_boxes = self.calibrate_box(total_boxes, reg[pick]) - total_boxes = self.convert_to_square(total_boxes) - total_boxes[:, 0:4] = np.round(total_boxes[:, 0:4]) + # nms + pick = nms(total_boxes, 0.7, 'Union') + total_boxes = total_boxes[pick] + total_boxes = self.calibrate_box(total_boxes, reg[pick]) + total_boxes = self.convert_to_square(total_boxes) + total_boxes[:, 0:4] = np.round(total_boxes[:, 0:4]) else: - total_boxes = np.array( [ [0.0, 0.0, img.shape[1], img.shape[0], 0.9] ] ,dtype=np.float32) + total_boxes = np.array( + [[0.0, 0.0, img.shape[1], img.shape[0], 0.9]], + dtype=np.float32) num_box = total_boxes.shape[0] - [dy, edy, dx, edx, y, ey, x, ex, tmpw, tmph] = self.pad(total_boxes, width, height) + [dy, edy, dx, edx, y, ey, x, ex, tmpw, + tmph] = self.pad(total_boxes, width, height) # (3, 48, 48) is the input shape for ONet input_buf = np.zeros((num_box, 3, 48, 48), dtype=np.float32) for i in range(num_box): tmp = np.zeros((tmph[i], tmpw[i], 3), dtype=np.float32) - tmp[dy[i]:edy[i]+1, dx[i]:edx[i]+1, :] = img[y[i]:ey[i]+1, x[i]:ex[i]+1, :] + tmp[dy[i]:edy[i] + 1, dx[i]:edx[i] + 1, :] = img[y[i]:ey[i] + 1, + x[i]:ex[i] + 1, :] input_buf[i, :, :, :] = adjust_input(cv2.resize(tmp, (48, 48))) output = self.ONet.predict(input_buf) @@ -241,22 +252,24 @@ class MtcnnDetector(object): if total_boxes.size == 0: return None - total_boxes[:, 4] = output[2][passed, 1].reshape((-1,)) + total_boxes[:, 4] = output[2][passed, 1].reshape((-1, )) reg = output[1][passed] points = output[0][passed] # compute landmark points bbw = total_boxes[:, 2] - total_boxes[:, 0] + 1 bbh = total_boxes[:, 3] - total_boxes[:, 1] + 1 - points[:, 0:5] = np.expand_dims(total_boxes[:, 0], 1) + np.expand_dims(bbw, 1) * points[:, 0:5] - points[:, 5:10] = np.expand_dims(total_boxes[:, 1], 1) + np.expand_dims(bbh, 1) * points[:, 5:10] + points[:, 0:5] = np.expand_dims( + total_boxes[:, 0], 1) + np.expand_dims(bbw, 1) * points[:, 0:5] + points[:, 5:10] = np.expand_dims( + total_boxes[:, 1], 1) + np.expand_dims(bbh, 1) * points[:, 5:10] # nms total_boxes = self.calibrate_box(total_boxes, reg) pick = nms(total_boxes, 0.7, 'Min') total_boxes = total_boxes[pick] points = points[pick] - + if not self.accurate_landmark: return total_boxes, points @@ -264,23 +277,27 @@ class MtcnnDetector(object): # extended stage ############################################# num_box = total_boxes.shape[0] - patchw = np.maximum(total_boxes[:, 2]-total_boxes[:, 0]+1, total_boxes[:, 3]-total_boxes[:, 1]+1) - patchw = np.round(patchw*0.25) + patchw = np.maximum(total_boxes[:, 2] - total_boxes[:, 0] + 1, + total_boxes[:, 3] - total_boxes[:, 1] + 1) + patchw = np.round(patchw * 0.25) # make it even - patchw[np.where(np.mod(patchw,2) == 1)] += 1 + patchw[np.where(np.mod(patchw, 2) == 1)] += 1 input_buf = np.zeros((num_box, 15, 24, 24), dtype=np.float32) for i in range(5): - x, y = points[:, i], points[:, i+5] - x, y = np.round(x-0.5*patchw), np.round(y-0.5*patchw) - [dy, edy, dx, edx, y, ey, x, ex, tmpw, tmph] = self.pad(np.vstack([x, y, x+patchw-1, y+patchw-1]).T, - width, - height) + x, y = points[:, i], points[:, i + 5] + x, y = np.round(x - 0.5 * patchw), np.round(y - 0.5 * patchw) + [dy, edy, dx, edx, y, ey, x, ex, tmpw, tmph] = self.pad( + np.vstack([x, y, x + patchw - 1, y + patchw - 1]).T, width, + height) for j in range(num_box): tmpim = np.zeros((tmpw[j], tmpw[j], 3), dtype=np.float32) - tmpim[dy[j]:edy[j]+1, dx[j]:edx[j]+1, :] = img[y[j]:ey[j]+1, x[j]:ex[j]+1, :] - input_buf[j, i*3:i*3+3, :, :] = adjust_input(cv2.resize(tmpim, (24, 24))) + tmpim[dy[j]:edy[j] + 1, + dx[j]:edx[j] + 1, :] = img[y[j]:ey[j] + 1, + x[j]:ex[j] + 1, :] + input_buf[j, i * 3:i * 3 + 3, :, :] = adjust_input( + cv2.resize(tmpim, (24, 24))) output = self.LNet.predict(input_buf) @@ -289,11 +306,13 @@ class MtcnnDetector(object): for k in range(5): # do not make a large movement - tmp_index = np.where(np.abs(output[k]-0.5) > 0.35) + tmp_index = np.where(np.abs(output[k] - 0.5) > 0.35) output[k][tmp_index[0]] = 0.5 - pointx[:, k] = np.round(points[:, k] - 0.5*patchw) + output[k][:, 0]*patchw - pointy[:, k] = np.round(points[:, k+5] - 0.5*patchw) + output[k][:, 1]*patchw + pointx[:, k] = np.round(points[:, k] - + 0.5 * patchw) + output[k][:, 0] * patchw + pointy[:, k] = np.round(points[:, k + 5] - + 0.5 * patchw) + output[k][:, 1] * patchw points = np.hstack([pointx, pointy]) points = points.astype(np.int32) @@ -317,7 +336,7 @@ class MtcnnDetector(object): # check input height, width, _ = img.shape - if det_type==0: + if det_type == 0: MIN_DET_SIZE = 12 if img is None: @@ -330,15 +349,15 @@ class MtcnnDetector(object): # detected boxes total_boxes = [] - minl = min( height, width) + minl = min(height, width) # get all the valid scales scales = [] - m = MIN_DET_SIZE/self.minsize + m = MIN_DET_SIZE / self.minsize minl *= m factor_count = 0 while minl > MIN_DET_SIZE: - scales.append(m*self.factor**factor_count) + scales.append(m * self.factor**factor_count) minl *= self.factor factor_count += 1 @@ -349,7 +368,7 @@ class MtcnnDetector(object): # return_boxes = self.detect_first_stage(img, scale, 0) # if return_boxes is not None: # total_boxes.append(return_boxes) - + sliced_index = self.slice_index(len(scales)) total_boxes = [] for batch in sliced_index: @@ -358,13 +377,13 @@ class MtcnnDetector(object): local_boxes = map( detect_first_stage_warpper, \ izip(repeat(img), self.PNets[:len(batch)], [scales[i] for i in batch], repeat(self.threshold[0])) ) total_boxes.extend(local_boxes) - - # remove the Nones - total_boxes = [ i for i in total_boxes if i is not None] + + # remove the Nones + total_boxes = [i for i in total_boxes if i is not None] if len(total_boxes) == 0: return None - + total_boxes = np.vstack(total_boxes) if total_boxes.size == 0: @@ -378,18 +397,20 @@ class MtcnnDetector(object): bbh = total_boxes[:, 3] - total_boxes[:, 1] + 1 # refine the bboxes - total_boxes = np.vstack([total_boxes[:, 0]+total_boxes[:, 5] * bbw, - total_boxes[:, 1]+total_boxes[:, 6] * bbh, - total_boxes[:, 2]+total_boxes[:, 7] * bbw, - total_boxes[:, 3]+total_boxes[:, 8] * bbh, - total_boxes[:, 4] - ]) + total_boxes = np.vstack([ + total_boxes[:, 0] + total_boxes[:, 5] * bbw, + total_boxes[:, 1] + total_boxes[:, 6] * bbh, + total_boxes[:, 2] + total_boxes[:, 7] * bbw, + total_boxes[:, 3] + total_boxes[:, 8] * bbh, total_boxes[:, 4] + ]) total_boxes = total_boxes.T total_boxes = self.convert_to_square(total_boxes) total_boxes[:, 0:4] = np.round(total_boxes[:, 0:4]) else: - total_boxes = np.array( [ [0.0, 0.0, img.shape[1], img.shape[0], 0.9] ] ,dtype=np.float32) + total_boxes = np.array( + [[0.0, 0.0, img.shape[1], img.shape[0], 0.9]], + dtype=np.float32) ############################################# # second stage @@ -397,13 +418,15 @@ class MtcnnDetector(object): num_box = total_boxes.shape[0] # pad the bbox - [dy, edy, dx, edx, y, ey, x, ex, tmpw, tmph] = self.pad(total_boxes, width, height) + [dy, edy, dx, edx, y, ey, x, ex, tmpw, + tmph] = self.pad(total_boxes, width, height) # (3, 24, 24) is the input shape for RNet input_buf = np.zeros((num_box, 3, 24, 24), dtype=np.float32) for i in range(num_box): tmp = np.zeros((tmph[i], tmpw[i], 3), dtype=np.uint8) - tmp[dy[i]:edy[i]+1, dx[i]:edx[i]+1, :] = img[y[i]:ey[i]+1, x[i]:ex[i]+1, :] + tmp[dy[i]:edy[i] + 1, dx[i]:edx[i] + 1, :] = img[y[i]:ey[i] + 1, + x[i]:ex[i] + 1, :] input_buf[i, :, :, :] = adjust_input(cv2.resize(tmp, (24, 24))) output = self.RNet.predict(input_buf) @@ -415,7 +438,7 @@ class MtcnnDetector(object): if total_boxes.size == 0: return None - total_boxes[:, 4] = output[1][passed, 1].reshape((-1,)) + total_boxes[:, 4] = output[1][passed, 1].reshape((-1, )) reg = output[0][passed] # nms @@ -431,13 +454,15 @@ class MtcnnDetector(object): num_box = total_boxes.shape[0] # pad the bbox - [dy, edy, dx, edx, y, ey, x, ex, tmpw, tmph] = self.pad(total_boxes, width, height) + [dy, edy, dx, edx, y, ey, x, ex, tmpw, + tmph] = self.pad(total_boxes, width, height) # (3, 48, 48) is the input shape for ONet input_buf = np.zeros((num_box, 3, 48, 48), dtype=np.float32) for i in range(num_box): tmp = np.zeros((tmph[i], tmpw[i], 3), dtype=np.float32) - tmp[dy[i]:edy[i]+1, dx[i]:edx[i]+1, :] = img[y[i]:ey[i]+1, x[i]:ex[i]+1, :] + tmp[dy[i]:edy[i] + 1, dx[i]:edx[i] + 1, :] = img[y[i]:ey[i] + 1, + x[i]:ex[i] + 1, :] input_buf[i, :, :, :] = adjust_input(cv2.resize(tmp, (48, 48))) output = self.ONet.predict(input_buf) @@ -449,22 +474,24 @@ class MtcnnDetector(object): if total_boxes.size == 0: return None - total_boxes[:, 4] = output[2][passed, 1].reshape((-1,)) + total_boxes[:, 4] = output[2][passed, 1].reshape((-1, )) reg = output[1][passed] points = output[0][passed] # compute landmark points bbw = total_boxes[:, 2] - total_boxes[:, 0] + 1 bbh = total_boxes[:, 3] - total_boxes[:, 1] + 1 - points[:, 0:5] = np.expand_dims(total_boxes[:, 0], 1) + np.expand_dims(bbw, 1) * points[:, 0:5] - points[:, 5:10] = np.expand_dims(total_boxes[:, 1], 1) + np.expand_dims(bbh, 1) * points[:, 5:10] + points[:, 0:5] = np.expand_dims( + total_boxes[:, 0], 1) + np.expand_dims(bbw, 1) * points[:, 0:5] + points[:, 5:10] = np.expand_dims( + total_boxes[:, 1], 1) + np.expand_dims(bbh, 1) * points[:, 5:10] # nms total_boxes = self.calibrate_box(total_boxes, reg) pick = nms(total_boxes, 0.7, 'Min') total_boxes = total_boxes[pick] points = points[pick] - + if not self.accurate_landmark: return total_boxes, points @@ -472,23 +499,27 @@ class MtcnnDetector(object): # extended stage ############################################# num_box = total_boxes.shape[0] - patchw = np.maximum(total_boxes[:, 2]-total_boxes[:, 0]+1, total_boxes[:, 3]-total_boxes[:, 1]+1) - patchw = np.round(patchw*0.25) + patchw = np.maximum(total_boxes[:, 2] - total_boxes[:, 0] + 1, + total_boxes[:, 3] - total_boxes[:, 1] + 1) + patchw = np.round(patchw * 0.25) # make it even - patchw[np.where(np.mod(patchw,2) == 1)] += 1 + patchw[np.where(np.mod(patchw, 2) == 1)] += 1 input_buf = np.zeros((num_box, 15, 24, 24), dtype=np.float32) for i in range(5): - x, y = points[:, i], points[:, i+5] - x, y = np.round(x-0.5*patchw), np.round(y-0.5*patchw) - [dy, edy, dx, edx, y, ey, x, ex, tmpw, tmph] = self.pad(np.vstack([x, y, x+patchw-1, y+patchw-1]).T, - width, - height) + x, y = points[:, i], points[:, i + 5] + x, y = np.round(x - 0.5 * patchw), np.round(y - 0.5 * patchw) + [dy, edy, dx, edx, y, ey, x, ex, tmpw, tmph] = self.pad( + np.vstack([x, y, x + patchw - 1, y + patchw - 1]).T, width, + height) for j in range(num_box): tmpim = np.zeros((tmpw[j], tmpw[j], 3), dtype=np.float32) - tmpim[dy[j]:edy[j]+1, dx[j]:edx[j]+1, :] = img[y[j]:ey[j]+1, x[j]:ex[j]+1, :] - input_buf[j, i*3:i*3+3, :, :] = adjust_input(cv2.resize(tmpim, (24, 24))) + tmpim[dy[j]:edy[j] + 1, + dx[j]:edx[j] + 1, :] = img[y[j]:ey[j] + 1, + x[j]:ex[j] + 1, :] + input_buf[j, i * 3:i * 3 + 3, :, :] = adjust_input( + cv2.resize(tmpim, (24, 24))) output = self.LNet.predict(input_buf) @@ -497,19 +528,19 @@ class MtcnnDetector(object): for k in range(5): # do not make a large movement - tmp_index = np.where(np.abs(output[k]-0.5) > 0.35) + tmp_index = np.where(np.abs(output[k] - 0.5) > 0.35) output[k][tmp_index[0]] = 0.5 - pointx[:, k] = np.round(points[:, k] - 0.5*patchw) + output[k][:, 0]*patchw - pointy[:, k] = np.round(points[:, k+5] - 0.5*patchw) + output[k][:, 1]*patchw + pointx[:, k] = np.round(points[:, k] - + 0.5 * patchw) + output[k][:, 0] * patchw + pointy[:, k] = np.round(points[:, k + 5] - + 0.5 * patchw) + output[k][:, 1] * patchw points = np.hstack([pointx, pointy]) points = points.astype(np.int32) return total_boxes, points - - def list2colmatrix(self, pts_list): """ convert list to column matrix @@ -542,15 +573,16 @@ class MtcnnDetector(object): tran_m: tran_b: """ - assert from_shape.shape[0] == to_shape.shape[0] and from_shape.shape[0] % 2 == 0 + assert from_shape.shape[0] == to_shape.shape[ + 0] and from_shape.shape[0] % 2 == 0 sigma_from = 0.0 sigma_to = 0.0 cov = np.matrix([[0.0, 0.0], [0.0, 0.0]]) # compute the mean and cov - from_shape_points = from_shape.reshape(from_shape.shape[0]/2, 2) - to_shape_points = to_shape.reshape(to_shape.shape[0]/2, 2) + from_shape_points = from_shape.reshape(from_shape.shape[0] / 2, 2) + to_shape_points = to_shape.reshape(to_shape.shape[0] / 2, 2) mean_from = from_shape_points.mean(axis=0) mean_to = to_shape_points.mean(axis=0) @@ -559,7 +591,8 @@ class MtcnnDetector(object): sigma_from += temp_dis * temp_dis temp_dis = np.linalg.norm(to_shape_points[i] - mean_to) sigma_to += temp_dis * temp_dis - cov += (to_shape_points[i].transpose() - mean_to.transpose()) * (from_shape_points[i] - mean_from) + cov += (to_shape_points[i].transpose() - + mean_to.transpose()) * (from_shape_points[i] - mean_from) sigma_from = sigma_from / to_shape_points.shape[0] sigma_to = sigma_to / to_shape_points.shape[0] @@ -601,27 +634,33 @@ class MtcnnDetector(object): """ crop_imgs = [] for p in points: - shape =[] - for k in range(len(p)/2): + shape = [] + for k in range(len(p) / 2): shape.append(p[k]) - shape.append(p[k+5]) + shape.append(p[k + 5]) if padding > 0: padding = padding else: padding = 0 # average positions of face points - mean_face_shape_x = [0.224152, 0.75610125, 0.490127, 0.254149, 0.726104] - mean_face_shape_y = [0.2119465, 0.2119465, 0.628106, 0.780233, 0.780233] + mean_face_shape_x = [ + 0.224152, 0.75610125, 0.490127, 0.254149, 0.726104 + ] + mean_face_shape_y = [ + 0.2119465, 0.2119465, 0.628106, 0.780233, 0.780233 + ] from_points = [] to_points = [] - for i in range(len(shape)/2): - x = (padding + mean_face_shape_x[i]) / (2 * padding + 1) * desired_size - y = (padding + mean_face_shape_y[i]) / (2 * padding + 1) * desired_size + for i in range(len(shape) / 2): + x = (padding + mean_face_shape_x[i]) / (2 * padding + + 1) * desired_size + y = (padding + mean_face_shape_y[i]) / (2 * padding + + 1) * desired_size to_points.append([x, y]) - from_points.append([shape[2*i], shape[2*i+1]]) + from_points.append([shape[2 * i], shape[2 * i + 1]]) # convert the points to Mat from_mat = self.list2colmatrix(from_points) @@ -634,9 +673,11 @@ class MtcnnDetector(object): probe_vec = tran_m * probe_vec scale = np.linalg.norm(probe_vec) - angle = 180.0 / math.pi * math.atan2(probe_vec[1, 0], probe_vec[0, 0]) + angle = 180.0 / math.pi * math.atan2(probe_vec[1, 0], probe_vec[0, + 0]) - from_center = [(shape[0]+shape[2])/2.0, (shape[1]+shape[3])/2.0] + from_center = [(shape[0] + shape[2]) / 2.0, + (shape[1] + shape[3]) / 2.0] to_center = [0, 0] to_center[1] = desired_size * 0.4 to_center[0] = desired_size * 0.5 @@ -644,7 +685,8 @@ class MtcnnDetector(object): ex = to_center[0] - from_center[0] ey = to_center[1] - from_center[1] - rot_mat = cv2.getRotationMatrix2D((from_center[0], from_center[1]), -1*angle, scale) + rot_mat = cv2.getRotationMatrix2D((from_center[0], from_center[1]), + -1 * angle, scale) rot_mat[0][2] += ex rot_mat[1][2] += ey @@ -652,4 +694,3 @@ class MtcnnDetector(object): crop_imgs.append(chips) return crop_imgs - diff --git a/gender-age/symbol_utils.py b/gender-age/symbol_utils.py index e23b01b..0a48697 100644 --- a/gender-age/symbol_utils.py +++ b/gender-age/symbol_utils.py @@ -1,5 +1,6 @@ import mxnet as mx + def Conv(**kwargs): #name = kwargs.get('name') #_weight = mx.symbol.Variable(name+'_weight') @@ -8,112 +9,250 @@ def Conv(**kwargs): body = mx.sym.Convolution(**kwargs) return body + def Act(data, act_type, name): - #ignore param act_type, set it in this function - body = mx.sym.LeakyReLU(data = data, act_type='prelu', name = name) + #ignore param act_type, set it in this function + body = mx.sym.LeakyReLU(data=data, act_type='prelu', name=name) return body + bn_mom = 0.9 -def Linear(data, num_filter=1, kernel=(1, 1), stride=(1, 1), pad=(0, 0), num_group=1, name=None, suffix=''): - conv = mx.sym.Convolution(data=data, num_filter=num_filter, kernel=kernel, num_group=num_group, stride=stride, pad=pad, no_bias=True, name='%s%s_conv2d' %(name, suffix)) - bn = mx.sym.BatchNorm(data=conv, name='%s%s_batchnorm' %(name, suffix), fix_gamma=False,momentum=bn_mom) + + +def Linear(data, + num_filter=1, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + num_group=1, + name=None, + suffix=''): + conv = mx.sym.Convolution(data=data, + num_filter=num_filter, + kernel=kernel, + num_group=num_group, + stride=stride, + pad=pad, + no_bias=True, + name='%s%s_conv2d' % (name, suffix)) + bn = mx.sym.BatchNorm(data=conv, + name='%s%s_batchnorm' % (name, suffix), + fix_gamma=False, + momentum=bn_mom) return bn + def get_fc1(last_conv, num_classes, fc_type): - bn_mom = 0.9 - body = last_conv - if fc_type=='Z': - body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn1') - body = mx.symbol.Dropout(data=body, p=0.4) - fc1 = body - elif fc_type=='E': - body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn1') - body = mx.symbol.Dropout(data=body, p=0.4) - fc1 = mx.sym.FullyConnected(data=body, num_hidden=num_classes, name='pre_fc1') - fc1 = mx.sym.BatchNorm(data=fc1, fix_gamma=True, eps=2e-5, momentum=bn_mom, name='fc1') - elif fc_type=='GAP': - bn1 = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn1') - relu1 = Act(data=bn1, act_type='relu', name='relu1') - # Although kernel is not used here when global_pool=True, we should put one - pool1 = mx.sym.Pooling(data=relu1, global_pool=True, kernel=(7, 7), pool_type='avg', name='pool1') - flat = mx.sym.Flatten(data=pool1) - fc1 = mx.sym.FullyConnected(data=flat, num_hidden=num_classes, name='pre_fc1') - fc1 = mx.sym.BatchNorm(data=fc1, fix_gamma=True, eps=2e-5, momentum=bn_mom, name='fc1') - elif fc_type=='GNAP': #mobilefacenet++ - filters_in = 512 # param in mobilefacenet - if num_classes>filters_in: - body = mx.sym.Convolution(data=last_conv, num_filter=num_classes, kernel=(1,1), stride=(1,1), pad=(0,0), no_bias=True, name='convx') - body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=0.9, name='convx_bn') - body = Act(data=body, act_type='relu', name='convx_relu') - filters_in = num_classes - else: - body = last_conv - body = mx.sym.BatchNorm(data=body, fix_gamma=True, eps=2e-5, momentum=0.9, name='bn6f') + bn_mom = 0.9 + body = last_conv + if fc_type == 'Z': + body = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name='bn1') + body = mx.symbol.Dropout(data=body, p=0.4) + fc1 = body + elif fc_type == 'E': + body = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name='bn1') + body = mx.symbol.Dropout(data=body, p=0.4) + fc1 = mx.sym.FullyConnected(data=body, + num_hidden=num_classes, + name='pre_fc1') + fc1 = mx.sym.BatchNorm(data=fc1, + fix_gamma=True, + eps=2e-5, + momentum=bn_mom, + name='fc1') + elif fc_type == 'GAP': + bn1 = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name='bn1') + relu1 = Act(data=bn1, act_type='relu', name='relu1') + # Although kernel is not used here when global_pool=True, we should put one + pool1 = mx.sym.Pooling(data=relu1, + global_pool=True, + kernel=(7, 7), + pool_type='avg', + name='pool1') + flat = mx.sym.Flatten(data=pool1) + fc1 = mx.sym.FullyConnected(data=flat, + num_hidden=num_classes, + name='pre_fc1') + fc1 = mx.sym.BatchNorm(data=fc1, + fix_gamma=True, + eps=2e-5, + momentum=bn_mom, + name='fc1') + elif fc_type == 'GNAP': #mobilefacenet++ + filters_in = 512 # param in mobilefacenet + if num_classes > filters_in: + body = mx.sym.Convolution(data=last_conv, + num_filter=num_classes, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + no_bias=True, + name='convx') + body = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=0.9, + name='convx_bn') + body = Act(data=body, act_type='relu', name='convx_relu') + filters_in = num_classes + else: + body = last_conv + body = mx.sym.BatchNorm(data=body, + fix_gamma=True, + eps=2e-5, + momentum=0.9, + name='bn6f') - spatial_norm=body*body - spatial_norm=mx.sym.sum(data=spatial_norm, axis=1, keepdims=True) - spatial_sqrt=mx.sym.sqrt(spatial_norm) - #spatial_mean=mx.sym.mean(spatial_sqrt, axis=(1,2,3), keepdims=True) - spatial_mean=mx.sym.mean(spatial_sqrt) - spatial_div_inverse=mx.sym.broadcast_div(spatial_mean, spatial_sqrt) + spatial_norm = body * body + spatial_norm = mx.sym.sum(data=spatial_norm, axis=1, keepdims=True) + spatial_sqrt = mx.sym.sqrt(spatial_norm) + #spatial_mean=mx.sym.mean(spatial_sqrt, axis=(1,2,3), keepdims=True) + spatial_mean = mx.sym.mean(spatial_sqrt) + spatial_div_inverse = mx.sym.broadcast_div(spatial_mean, spatial_sqrt) - spatial_attention_inverse=mx.symbol.tile(spatial_div_inverse, reps=(1,filters_in,1,1)) - body=body*spatial_attention_inverse - #body = mx.sym.broadcast_mul(body, spatial_div_inverse) + spatial_attention_inverse = mx.symbol.tile(spatial_div_inverse, + reps=(1, filters_in, 1, 1)) + body = body * spatial_attention_inverse + #body = mx.sym.broadcast_mul(body, spatial_div_inverse) - fc1 = mx.sym.Pooling(body, kernel=(7, 7), global_pool=True, pool_type='avg') - if num_classes1: - if fc_type[1]=='X': - print('dropout mode') - flat = mx.symbol.Dropout(data=flat, p=0.2) - fc_type = fc_type[0] - if fc_type=='A': - fc1 = flat - else: - #B-D - #B - fc1 = mx.sym.FullyConnected(data=flat, num_hidden=num_classes, name='pre_fc1') - if fc_type=='C': - fc1 = mx.sym.BatchNorm(data=fc1, fix_gamma=True, eps=2e-5, momentum=bn_mom, name='fc1') - elif fc_type=='D': - fc1 = mx.sym.BatchNorm(data=fc1, fix_gamma=True, eps=2e-5, momentum=bn_mom, name='fc1') - fc1 = Act(data=fc1, act_type='relu', name='fc1_relu') - return fc1 + bn1 = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name='bn1') + relu1 = Act(data=bn1, act_type='relu', name='relu1') + # Although kernel is not used here when global_pool=True, we should put one + pool1 = mx.sym.Pooling(data=relu1, + global_pool=True, + kernel=(7, 7), + pool_type='avg', + name='pool1') + flat = mx.sym.Flatten(data=pool1) + if len(fc_type) > 1: + if fc_type[1] == 'X': + print('dropout mode') + flat = mx.symbol.Dropout(data=flat, p=0.2) + fc_type = fc_type[0] + if fc_type == 'A': + fc1 = flat + else: + #B-D + #B + fc1 = mx.sym.FullyConnected(data=flat, + num_hidden=num_classes, + name='pre_fc1') + if fc_type == 'C': + fc1 = mx.sym.BatchNorm(data=fc1, + fix_gamma=True, + eps=2e-5, + momentum=bn_mom, + name='fc1') + elif fc_type == 'D': + fc1 = mx.sym.BatchNorm(data=fc1, + fix_gamma=True, + eps=2e-5, + momentum=bn_mom, + name='fc1') + fc1 = Act(data=fc1, act_type='relu', name='fc1_relu') + return fc1 + def residual_unit_v3(data, num_filter, stride, dim_match, name, **kwargs): - """Return ResNet Unit symbol for building ResNet Parameters ---------- @@ -136,21 +275,54 @@ def residual_unit_v3(data, num_filter, stride, dim_match, name, **kwargs): workspace = kwargs.get('workspace', 256) memonger = kwargs.get('memonger', False) #print('in unit3') - bn1 = mx.sym.BatchNorm(data=data, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn1') - conv1 = Conv(data=bn1, num_filter=num_filter, kernel=(3,3), stride=(1,1), pad=(1,1), - no_bias=True, workspace=workspace, name=name + '_conv1') - bn2 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn2') + bn1 = mx.sym.BatchNorm(data=data, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn1') + conv1 = Conv(data=bn1, + num_filter=num_filter, + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv1') + bn2 = mx.sym.BatchNorm(data=conv1, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn2') act1 = Act(data=bn2, act_type='relu', name=name + '_relu1') - conv2 = Conv(data=act1, num_filter=num_filter, kernel=(3,3), stride=stride, pad=(1,1), - no_bias=True, workspace=workspace, name=name + '_conv2') - bn3 = mx.sym.BatchNorm(data=conv2, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn3') + conv2 = Conv(data=act1, + num_filter=num_filter, + kernel=(3, 3), + stride=stride, + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv2') + bn3 = mx.sym.BatchNorm(data=conv2, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn3') if dim_match: shortcut = data else: - conv1sc = Conv(data=data, num_filter=num_filter, kernel=(1,1), stride=stride, no_bias=True, - workspace=workspace, name=name+'_conv1sc') - shortcut = mx.sym.BatchNorm(data=conv1sc, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_sc') + conv1sc = Conv(data=data, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + no_bias=True, + workspace=workspace, + name=name + '_conv1sc') + shortcut = mx.sym.BatchNorm(data=conv1sc, + fix_gamma=False, + momentum=bn_mom, + eps=2e-5, + name=name + '_sc') if memonger: shortcut._set_attr(mirror_stage='True') return bn3 + shortcut @@ -159,24 +331,50 @@ def residual_unit_v3(data, num_filter, stride, dim_match, name, **kwargs): def get_head(data, version_input, num_filter): bn_mom = 0.9 workspace = 256 - kwargs = {'bn_mom': bn_mom, 'workspace' : workspace} - data = data-127.5 - data = data*0.0078125 + kwargs = {'bn_mom': bn_mom, 'workspace': workspace} + data = data - 127.5 + data = data * 0.0078125 #data = mx.sym.BatchNorm(data=data, fix_gamma=True, eps=2e-5, momentum=bn_mom, name='bn_data') - if version_input==0: - body = Conv(data=data, num_filter=num_filter, kernel=(7, 7), stride=(2,2), pad=(3, 3), - no_bias=True, name="conv0", workspace=workspace) - body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn0') - body = Act(data=body, act_type='relu', name='relu0') - body = mx.sym.Pooling(data=body, kernel=(3, 3), stride=(2,2), pad=(1,1), pool_type='max') + if version_input == 0: + body = Conv(data=data, + num_filter=num_filter, + kernel=(7, 7), + stride=(2, 2), + pad=(3, 3), + no_bias=True, + name="conv0", + workspace=workspace) + body = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name='bn0') + body = Act(data=body, act_type='relu', name='relu0') + body = mx.sym.Pooling(data=body, + kernel=(3, 3), + stride=(2, 2), + pad=(1, 1), + pool_type='max') else: - body = data - _num_filter = min(num_filter, 64) - body = Conv(data=body, num_filter=_num_filter, kernel=(3,3), stride=(1,1), pad=(1, 1), - no_bias=True, name="conv0", workspace=workspace) - body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn0') - body = Act(data=body, act_type='relu', name='relu0') - body = residual_unit_v3(body, _num_filter, (2, 2), False, name='head', **kwargs) + body = data + _num_filter = min(num_filter, 64) + body = Conv(data=body, + num_filter=_num_filter, + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + name="conv0", + workspace=workspace) + body = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name='bn0') + body = Act(data=body, act_type='relu', name='relu0') + body = residual_unit_v3(body, + _num_filter, (2, 2), + False, + name='head', + **kwargs) return body - - diff --git a/gender-age/test.py b/gender-age/test.py index 663a8ab..78fc3b0 100644 --- a/gender-age/test.py +++ b/gender-age/test.py @@ -9,9 +9,15 @@ parser = argparse.ArgumentParser(description='face model test') # general parser.add_argument('--image-size', default='112,112', help='') parser.add_argument('--image', default='Tom_Hanks_54745.png', help='') -parser.add_argument('--model', default='model/model,0', help='path to load model.') +parser.add_argument('--model', + default='model/model,0', + help='path to load model.') parser.add_argument('--gpu', default=0, type=int, help='gpu id') -parser.add_argument('--det', default=0, type=int, help='mtcnn option, 1 means using R+O, 0 means detect from begining') +parser.add_argument( + '--det', + default=0, + type=int, + help='mtcnn option, 1 means using R+O, 0 means detect from begining') args = parser.parse_args() model = face_model.FaceModel(args) @@ -21,14 +27,13 @@ img = model.get_input(img) #f1 = model.get_feature(img) #print(f1[0:10]) for _ in range(5): - gender, age = model.get_ga(img) + gender, age = model.get_ga(img) time_now = datetime.datetime.now() count = 200 for _ in range(count): - gender, age = model.get_ga(img) + gender, age = model.get_ga(img) time_now2 = datetime.datetime.now() diff = time_now2 - time_now -print('time cost', diff.total_seconds()/count) -print('gender is',gender) +print('time cost', diff.total_seconds() / count) +print('gender is', gender) print('age is', age) - diff --git a/gender-age/train.py b/gender-age/train.py index effeadb..4948d78 100644 --- a/gender-age/train.py +++ b/gender-age/train.py @@ -18,305 +18,396 @@ import mxnet.optimizer as optimizer sys.path.append(os.path.join(os.path.dirname(__file__), 'common')) #import face_image import fresnet -import fmobilenet - +import fmobilenet logger = logging.getLogger() logger.setLevel(logging.INFO) -AGE=100 - +AGE = 100 args = None class AccMetric(mx.metric.EvalMetric): - def __init__(self): - self.axis = 1 - super(AccMetric, self).__init__( - 'acc', axis=self.axis, - output_names=None, label_names=None) - self.losses = [] - self.count = 0 + def __init__(self): + self.axis = 1 + super(AccMetric, self).__init__('acc', + axis=self.axis, + output_names=None, + label_names=None) + self.losses = [] + self.count = 0 + + def update(self, labels, preds): + self.count += 1 + label = labels[0].asnumpy()[:, 0:1] + pred_label = preds[-1].asnumpy()[:, 0:2] + pred_label = np.argmax(pred_label, axis=self.axis) + pred_label = pred_label.astype('int32').flatten() + label = label.astype('int32').flatten() + assert label.shape == pred_label.shape + self.sum_metric += (pred_label.flat == label.flat).sum() + self.num_inst += len(pred_label.flat) - def update(self, labels, preds): - self.count+=1 - label = labels[0].asnumpy()[:,0:1] - pred_label = preds[-1].asnumpy()[:,0:2] - pred_label = np.argmax(pred_label, axis=self.axis) - pred_label = pred_label.astype('int32').flatten() - label = label.astype('int32').flatten() - assert label.shape==pred_label.shape - self.sum_metric += (pred_label.flat == label.flat).sum() - self.num_inst += len(pred_label.flat) class LossValueMetric(mx.metric.EvalMetric): - def __init__(self): - self.axis = 1 - super(LossValueMetric, self).__init__( - 'lossvalue', axis=self.axis, - output_names=None, label_names=None) - self.losses = [] + def __init__(self): + self.axis = 1 + super(LossValueMetric, self).__init__('lossvalue', + axis=self.axis, + output_names=None, + label_names=None) + self.losses = [] + + def update(self, labels, preds): + loss = preds[-1].asnumpy()[0] + self.sum_metric += loss + self.num_inst += 1.0 + gt_label = preds[-2].asnumpy() + #print(gt_label) - def update(self, labels, preds): - loss = preds[-1].asnumpy()[0] - self.sum_metric += loss - self.num_inst += 1.0 - gt_label = preds[-2].asnumpy() - #print(gt_label) class MAEMetric(mx.metric.EvalMetric): - def __init__(self): - self.axis = 1 - super(MAEMetric, self).__init__( - 'MAE', axis=self.axis, - output_names=None, label_names=None) - self.losses = [] - self.count = 0 + def __init__(self): + self.axis = 1 + super(MAEMetric, self).__init__('MAE', + axis=self.axis, + output_names=None, + label_names=None) + self.losses = [] + self.count = 0 + + def update(self, labels, preds): + self.count += 1 + label = labels[0].asnumpy() + label_age = np.count_nonzero(label[:, 1:], axis=1) + pred_age = np.zeros(label_age.shape, dtype=np.int) + #pred_age = np.zeros( label_age.shape, dtype=np.float32) + pred = preds[-1].asnumpy() + for i in range(AGE): + _pred = pred[:, 2 + i * 2:4 + i * 2] + _pred = np.argmax(_pred, axis=1) + #pred = pred[:,1] + pred_age += _pred + #pred_age = pred_age.astype(np.int) + mae = np.mean(np.abs(label_age - pred_age)) + self.sum_metric += mae + self.num_inst += 1.0 - def update(self, labels, preds): - self.count+=1 - label = labels[0].asnumpy() - label_age = np.count_nonzero(label[:,1:], axis=1) - pred_age = np.zeros( label_age.shape, dtype=np.int) - #pred_age = np.zeros( label_age.shape, dtype=np.float32) - pred = preds[-1].asnumpy() - for i in range(AGE): - _pred = pred[:,2+i*2:4+i*2] - _pred = np.argmax(_pred, axis=1) - #pred = pred[:,1] - pred_age += _pred - #pred_age = pred_age.astype(np.int) - mae = np.mean(np.abs(label_age - pred_age)) - self.sum_metric += mae - self.num_inst += 1.0 class CUMMetric(mx.metric.EvalMetric): - def __init__(self, n=5): - self.axis = 1 - self.n = n - super(CUMMetric, self).__init__( - 'CUM_%d'%n, axis=self.axis, - output_names=None, label_names=None) - self.losses = [] - self.count = 0 + def __init__(self, n=5): + self.axis = 1 + self.n = n + super(CUMMetric, self).__init__('CUM_%d' % n, + axis=self.axis, + output_names=None, + label_names=None) + self.losses = [] + self.count = 0 + + def update(self, labels, preds): + self.count += 1 + label = labels[0].asnumpy() + label_age = np.count_nonzero(label[:, 1:], axis=1) + pred_age = np.zeros(label_age.shape, dtype=np.int) + pred = preds[-1].asnumpy() + for i in range(AGE): + _pred = pred[:, 2 + i * 2:4 + i * 2] + _pred = np.argmax(_pred, axis=1) + #pred = pred[:,1] + pred_age += _pred + diff = np.abs(label_age - pred_age) + cum = np.sum((diff < self.n)) + self.sum_metric += cum + self.num_inst += len(label_age) - def update(self, labels, preds): - self.count+=1 - label = labels[0].asnumpy() - label_age = np.count_nonzero(label[:,1:], axis=1) - pred_age = np.zeros( label_age.shape, dtype=np.int) - pred = preds[-1].asnumpy() - for i in range(AGE): - _pred = pred[:,2+i*2:4+i*2] - _pred = np.argmax(_pred, axis=1) - #pred = pred[:,1] - pred_age += _pred - diff = np.abs(label_age - pred_age) - cum = np.sum( (diff0: - for i in range(len(cvd.split(','))): - ctx.append(mx.gpu(i)) - if len(ctx)==0: - ctx = [mx.cpu()] - print('use cpu') + if len(cvd) > 0: + for i in range(len(cvd.split(','))): + ctx.append(mx.gpu(i)) + if len(ctx) == 0: + ctx = [mx.cpu()] + print('use cpu') else: - print('gpu num:', len(ctx)) + print('gpu num:', len(ctx)) prefix = args.prefix prefix_dir = os.path.dirname(prefix) if not os.path.exists(prefix_dir): - os.makedirs(prefix_dir) + os.makedirs(prefix_dir) end_epoch = args.end_epoch args.ctx_num = len(ctx) args.num_layers = int(args.network[1:]) print('num_layers', args.num_layers) - if args.per_batch_size==0: - args.per_batch_size = 128 - args.batch_size = args.per_batch_size*args.ctx_num + if args.per_batch_size == 0: + args.per_batch_size = 128 + args.batch_size = args.per_batch_size * args.ctx_num args.rescale_threshold = 0 args.image_channel = 3 data_dir_list = args.data_dir.split(',') - assert len(data_dir_list)==1 + assert len(data_dir_list) == 1 data_dir = data_dir_list[0] path_imgrec = None path_imglist = None image_size = [int(x) for x in args.image_size.split(',')] - assert len(image_size)==2 - assert image_size[0]==image_size[1] + assert len(image_size) == 2 + assert image_size[0] == image_size[1] args.image_h = image_size[0] args.image_w = image_size[1] print('image_size', image_size) path_imgrec = os.path.join(data_dir, "train.rec") path_imgrec_val = os.path.join(data_dir, "val.rec") - print('Called with argument:', args) - data_shape = (args.image_channel,image_size[0],image_size[1]) + data_shape = (args.image_channel, image_size[0], image_size[1]) mean = None begin_epoch = 0 base_lr = args.lr base_wd = args.wd base_mom = args.mom - if len(args.pretrained)==0: - arg_params = None - aux_params = None - sym, arg_params, aux_params = get_symbol(args, arg_params, aux_params) + if len(args.pretrained) == 0: + arg_params = None + aux_params = None + sym, arg_params, aux_params = get_symbol(args, arg_params, aux_params) else: - vec = args.pretrained.split(',') - print('loading', vec) - _, arg_params, aux_params = mx.model.load_checkpoint(vec[0], int(vec[1])) - sym, arg_params, aux_params = get_symbol(args, arg_params, aux_params) + vec = args.pretrained.split(',') + print('loading', vec) + _, arg_params, aux_params = mx.model.load_checkpoint( + vec[0], int(vec[1])) + sym, arg_params, aux_params = get_symbol(args, arg_params, aux_params) #label_name = 'softmax_label' #label_shape = (args.batch_size,) model = mx.mod.Module( - context = ctx, - symbol = sym, + context=ctx, + symbol=sym, ) val_dataiter = None train_dataiter = FaceImageIter( - batch_size = args.batch_size, - data_shape = data_shape, - path_imgrec = path_imgrec, - shuffle = True, - rand_mirror = args.rand_mirror, - mean = mean, - cutoff = args.cutoff, - color_jittering = args.color, + batch_size=args.batch_size, + data_shape=data_shape, + path_imgrec=path_imgrec, + shuffle=True, + rand_mirror=args.rand_mirror, + mean=mean, + cutoff=args.cutoff, + color_jittering=args.color, ) val_dataiter = FaceImageIter( - batch_size = args.batch_size, - data_shape = data_shape, - path_imgrec = path_imgrec_val, - shuffle = False, - rand_mirror = False, - mean = mean, + batch_size=args.batch_size, + data_shape=data_shape, + path_imgrec=path_imgrec_val, + shuffle=False, + rand_mirror=False, + mean=mean, ) - metric = mx.metric.CompositeEvalMetric([AccMetric(), MAEMetric(), CUMMetric()]) + metric = mx.metric.CompositeEvalMetric( + [AccMetric(), MAEMetric(), CUMMetric()]) - if args.network[0]=='r' or args.network[0]=='y': - initializer = mx.init.Xavier(rnd_type='gaussian', factor_type="out", magnitude=2) #resnet style - elif args.network[0]=='i' or args.network[0]=='x': - initializer = mx.init.Xavier(rnd_type='gaussian', factor_type="in", magnitude=2) #inception + if args.network[0] == 'r' or args.network[0] == 'y': + initializer = mx.init.Xavier(rnd_type='gaussian', + factor_type="out", + magnitude=2) #resnet style + elif args.network[0] == 'i' or args.network[0] == 'x': + initializer = mx.init.Xavier(rnd_type='gaussian', + factor_type="in", + magnitude=2) #inception else: - initializer = mx.init.Xavier(rnd_type='uniform', factor_type="in", magnitude=2) - _rescale = 1.0/args.ctx_num - opt = optimizer.SGD(learning_rate=base_lr, momentum=base_mom, wd=base_wd, rescale_grad=_rescale) + initializer = mx.init.Xavier(rnd_type='uniform', + factor_type="in", + magnitude=2) + _rescale = 1.0 / args.ctx_num + opt = optimizer.SGD(learning_rate=base_lr, + momentum=base_mom, + wd=base_wd, + rescale_grad=_rescale) #opt = optimizer.Nadam(learning_rate=base_lr, wd=base_wd, rescale_grad=_rescale) som = 20 _cb = mx.callback.Speedometer(args.batch_size, som) lr_steps = [int(x) for x in args.lr_steps.split(',')] global_step = [0] + def _batch_callback(param): - _cb(param) - global_step[0]+=1 - mbatch = global_step[0] - for _lr in lr_steps: - if mbatch==_lr: - opt.lr *= 0.1 - print('lr change to', opt.lr) - break - if mbatch%1000==0: - print('lr-batch-epoch:',opt.lr,param.nbatch,param.epoch) - if mbatch==lr_steps[-1]: - arg, aux = model.get_params() - all_layers = model.symbol.get_internals() - _sym = all_layers['fc1_output'] - mx.model.save_checkpoint(args.prefix, 0, _sym, arg, aux) - sys.exit(0) + _cb(param) + global_step[0] += 1 + mbatch = global_step[0] + for _lr in lr_steps: + if mbatch == _lr: + opt.lr *= 0.1 + print('lr change to', opt.lr) + break + if mbatch % 1000 == 0: + print('lr-batch-epoch:', opt.lr, param.nbatch, param.epoch) + if mbatch == lr_steps[-1]: + arg, aux = model.get_params() + all_layers = model.symbol.get_internals() + _sym = all_layers['fc1_output'] + mx.model.save_checkpoint(args.prefix, 0, _sym, arg, aux) + sys.exit(0) epoch_cb = None train_dataiter = mx.io.PrefetchingIter(train_dataiter) print('start fitting') - model.fit(train_dataiter, - begin_epoch = begin_epoch, - num_epoch = end_epoch, - eval_data = val_dataiter, - eval_metric = metric, - kvstore = 'device', - optimizer = opt, + model.fit( + train_dataiter, + begin_epoch=begin_epoch, + num_epoch=end_epoch, + eval_data=val_dataiter, + eval_metric=metric, + kvstore='device', + optimizer=opt, #optimizer_params = optimizer_params, - initializer = initializer, - arg_params = arg_params, - aux_params = aux_params, - allow_missing = True, - batch_end_callback = _batch_callback, - epoch_end_callback = epoch_cb ) + initializer=initializer, + arg_params=arg_params, + aux_params=aux_params, + allow_missing=True, + batch_end_callback=_batch_callback, + epoch_end_callback=epoch_cb) + def main(): #time.sleep(3600*6.5) @@ -324,6 +415,6 @@ def main(): args = parse_args() train_net(args) + if __name__ == '__main__': main() - diff --git a/iccv19-challenge/gen_image_feature.py b/iccv19-challenge/gen_image_feature.py deleted file mode 100644 index 55453ac..0000000 --- a/iccv19-challenge/gen_image_feature.py +++ /dev/null @@ -1,153 +0,0 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os - -from datetime import datetime -import os.path -from easydict import EasyDict as edict -import time -import json -import sys -import numpy as np -import importlib -import itertools -import argparse -import struct -import cv2 -import sklearn -from sklearn.preprocessing import normalize -import mxnet as mx -from mxnet import ndarray as nd - -image_shape = None -net = None -data_size = 1862120 -emb_size = 0 -use_flip = True - - - -def do_flip(data): - for idx in range(data.shape[0]): - data[idx,:,:] = np.fliplr(data[idx,:,:]) - -def get_feature(buffer): - global emb_size - if use_flip: - input_blob = np.zeros( (len(buffer)*2, 3, image_shape[1], image_shape[2] ) ) - else: - input_blob = np.zeros( (len(buffer), 3, image_shape[1], image_shape[2] ) ) - idx = 0 - for item in buffer: - img = cv2.imread(item)[:,:,::-1] #to rgb - img = np.transpose( img, (2,0,1) ) - attempts = [0,1] if use_flip else [0] - for flipid in attempts: - _img = np.copy(img) - if flipid==1: - do_flip(_img) - input_blob[idx] = _img - idx+=1 - data = mx.nd.array(input_blob) - db = mx.io.DataBatch(data=(data,)) - net.model.forward(db, is_train=False) - _embedding = net.model.get_outputs()[0].asnumpy() - if emb_size==0: - emb_size = _embedding.shape[1] - print('set emb_size to ', emb_size) - embedding = np.zeros( (len(buffer), emb_size), dtype=np.float32 ) - if use_flip: - embedding1 = _embedding[0::2] - embedding2 = _embedding[1::2] - embedding = embedding1+embedding2 - else: - embedding = _embedding - embedding = sklearn.preprocessing.normalize(embedding) - return embedding - - - -def write_bin(path, m): - rows, cols = m.shape - with open(path, 'wb') as f: - f.write(struct.pack('4i', rows,cols,cols*4,5)) - f.write(m.data) - -def main(args): - global image_shape - global net - - print(args) - ctx = [] - cvd = os.environ['CUDA_VISIBLE_DEVICES'].strip() - if len(cvd)>0: - for i in range(len(cvd.split(','))): - ctx.append(mx.gpu(i)) - if len(ctx)==0: - ctx = [mx.cpu()] - print('use cpu') - else: - print('gpu num:', len(ctx)) - image_shape = [int(x) for x in args.image_size.split(',')] - vec = args.model.split(',') - assert len(vec)>1 - prefix = vec[0] - epoch = int(vec[1]) - print('loading',prefix, epoch) - net = edict() - net.ctx = ctx - net.sym, net.arg_params, net.aux_params = mx.model.load_checkpoint(prefix, epoch) - #net.arg_params, net.aux_params = ch_dev(net.arg_params, net.aux_params, net.ctx) - all_layers = net.sym.get_internals() - net.sym = all_layers['fc1_output'] - net.model = mx.mod.Module(symbol=net.sym, context=net.ctx, label_names = None) - net.model.bind(data_shapes=[('data', (args.batch_size, 3, image_shape[1], image_shape[2]))]) - net.model.set_params(net.arg_params, net.aux_params) - - features_all = None - - i = 0 - fstart = 0 - buffer = [] - for line in open(os.path.join(args.input, 'filelist.txt'), 'r'): - if i%1000==0: - print("processing ",i) - i+=1 - line = line.strip() - image_path = os.path.join(args.input, line) - buffer.append(image_path) - if len(buffer)==args.batch_size: - embedding = get_feature(buffer) - buffer = [] - fend = fstart+embedding.shape[0] - if features_all is None: - features_all = np.zeros( (data_size, emb_size), dtype=np.float32 ) - #print('writing', fstart, fend) - features_all[fstart:fend,:] = embedding - fstart = fend - if len(buffer)>0: - embedding = get_feature(buffer) - fend = fstart+embedding.shape[0] - print('writing', fstart, fend) - features_all[fstart:fend,:] = embedding - write_bin(args.output, features_all) - #os.system("bypy upload %s"%args.output) - - - -def parse_arguments(argv): - parser = argparse.ArgumentParser() - - parser.add_argument('--batch_size', type=int, help='', default=32) - parser.add_argument('--image_size', type=str, help='', default='3,112,112') - parser.add_argument('--input', type=str, help='', default='') - parser.add_argument('--output', type=str, help='', default='') - parser.add_argument('--model', type=str, help='', default='') - return parser.parse_args(argv) - -if __name__ == '__main__': - main(parse_arguments(sys.argv[1:])) - - diff --git a/iccv19-challenge/gen_video_feature.py b/iccv19-challenge/gen_video_feature.py deleted file mode 100644 index 14f4a14..0000000 --- a/iccv19-challenge/gen_video_feature.py +++ /dev/null @@ -1,204 +0,0 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os - -from datetime import datetime -import os.path -from easydict import EasyDict as edict -import time -import json -import glob -import sys -import numpy as np -import importlib -import itertools -import argparse -import struct -import cv2 -import sklearn -from sklearn.preprocessing import normalize -import mxnet as mx -from mxnet import ndarray as nd - -image_shape = None -net = None -data_size = 203848 -emb_size = 0 -use_flip = False -ctx_num = 0 - - - -def do_flip(data): - for idx in range(data.shape[0]): - data[idx,:,:] = np.fliplr(data[idx,:,:]) - -def get_feature(buffer): - global emb_size - input_count = len(buffer) - if use_flip: - input_count *= 2 - network_count = input_count - if input_count%ctx_num!=0: - network_count = (input_count//ctx_num+1)*ctx_num - - input_blob = np.zeros( (network_count, 3, image_shape[1], image_shape[2]), dtype=np.float32) - idx = 0 - for item in buffer: - img = cv2.imread(item)[:,:,::-1] #to rgb - img = np.transpose( img, (2,0,1) ) - attempts = [0,1] if use_flip else [0] - for flipid in attempts: - _img = np.copy(img) - if flipid==1: - do_flip(_img) - input_blob[idx] = _img - idx+=1 - data = mx.nd.array(input_blob) - db = mx.io.DataBatch(data=(data,)) - net.model.forward(db, is_train=False) - _embedding = net.model.get_outputs()[0].asnumpy() - _embedding = _embedding[0:input_count] - if emb_size==0: - emb_size = _embedding.shape[1] - print('set emb_size to ', emb_size) - embedding = np.zeros( (len(buffer), emb_size), dtype=np.float32 ) - if use_flip: - embedding1 = _embedding[0::2] - embedding2 = _embedding[1::2] - embedding = embedding1+embedding2 - else: - embedding = _embedding - embedding = sklearn.preprocessing.normalize(embedding) - return embedding - -def write_bin(path, m): - rows, cols = m.shape - with open(path, 'wb') as f: - f.write(struct.pack('4i', rows,cols,cols*4,5)) - f.write(m.data) - -def main(args): - global image_shape - global net - global ctx_num - - print(args) - ctx = [] - cvd = os.environ['CUDA_VISIBLE_DEVICES'].strip() - if len(cvd)>0: - for i in range(len(cvd.split(','))): - ctx.append(mx.gpu(i)) - if len(ctx)==0: - ctx = [mx.cpu()] - print('use cpu') - else: - print('gpu num:', len(ctx)) - ctx_num = len(ctx) - image_shape = [int(x) for x in args.image_size.split(',')] - vec = args.model.split(',') - assert len(vec)>1 - prefix = vec[0] - epoch = int(vec[1]) - print('loading',prefix, epoch) - net = edict() - net.ctx = ctx - net.sym, net.arg_params, net.aux_params = mx.model.load_checkpoint(prefix, epoch) - #net.arg_params, net.aux_params = ch_dev(net.arg_params, net.aux_params, net.ctx) - all_layers = net.sym.get_internals() - net.sym = all_layers['fc1_output'] - net.model = mx.mod.Module(symbol=net.sym, context=net.ctx, label_names = None) - net.model.bind(data_shapes=[('data', (args.batch_size, 3, image_shape[1], image_shape[2]))]) - net.model.set_params(net.arg_params, net.aux_params) - - features_all = None - - i = 0 - filelist = os.path.join(args.input, 'filelist.txt') - #print(filelist) - buffer_images = [] - buffer_embedding = np.zeros( (0,0), dtype=np.float32) - aggr_nums = [] - row_idx = 0 - for line in open(filelist, 'r'): - if i%1000==0: - print("processing ",i) - i+=1 - #print('stat', i, len(buffer_images), buffer_embedding.shape, aggr_nums, row_idx) - videoname = line.strip().split()[0] - images = glob.glob("%s/%s/*.jpg"%(args.input, videoname)) - assert len(images)>0 - image_features = [] - for image_path in images: - buffer_images.append(image_path) - aggr_nums.append(len(images)) - while len(buffer_images)>=args.batch_size: - embedding = get_feature(buffer_images[0:args.batch_size]) - buffer_images = buffer_images[args.batch_size:] - if buffer_embedding.shape[1]==0: - buffer_embedding = embedding.copy() - else: - buffer_embedding = np.concatenate( (buffer_embedding, embedding), axis=0) - buffer_idx = 0 - acount = 0 - for anum in aggr_nums: - if buffer_embedding.shape[0]>=anum+buffer_idx: - image_features = buffer_embedding[buffer_idx:buffer_idx+anum] - video_feature = np.sum(image_features, axis=0, keepdims=True) - video_feature = sklearn.preprocessing.normalize(video_feature) - if features_all is None: - features_all = np.zeros( (data_size, video_feature.shape[1]), dtype=np.float32 ) - #print('write to', row_idx, anum, buffer_embedding.shape) - features_all[row_idx] = video_feature.flatten() - row_idx+=1 - buffer_idx += anum - acount+=1 - else: - break - aggr_nums = aggr_nums[acount:] - buffer_embedding = buffer_embedding[buffer_idx:] - - if len(buffer_images)>0: - embedding = get_feature(buffer_images) - buffer_images = buffer_images[args.batch_size:] - buffer_embedding = np.concatenate( (buffer_embedding, embedding), axis=0) - buffer_idx = 0 - acount = 0 - for anum in aggr_nums: - assert buffer_embedding.shape[0]>=anum+buffer_idx - image_features = buffer_embedding[buffer_idx:buffer_idx+anum] - video_feature = np.sum(image_features, axis=0, keepdims=True) - video_feature = sklearn.preprocessing.normalize(video_feature) - #print('last write to', row_idx, anum, buffer_embedding.shape) - features_all[row_idx] = video_feature.flatten() - row_idx+=1 - buffer_idx += anum - acount+=1 - - aggr_nums = aggr_nums[acount:] - buffer_embedding = buffer_embedding[buffer_idx:] - assert len(aggr_nums)==0 - assert buffer_embedding.shape[0]==0 - - write_bin(args.output, features_all) - print(row_idx, features_all.shape) - #os.system("bypy upload %s"%args.output) - - - -def parse_arguments(argv): - parser = argparse.ArgumentParser() - - parser.add_argument('--batch_size', type=int, help='', default=32) - parser.add_argument('--image_size', type=str, help='', default='3,112,112') - parser.add_argument('--input', type=str, help='', default='./testdata-video') - parser.add_argument('--output', type=str, help='', default='') - parser.add_argument('--model', type=str, help='', default='') - return parser.parse_args(argv) - -if __name__ == '__main__': - main(parse_arguments(sys.argv[1:])) - - diff --git a/models/README.md b/models/README.md deleted file mode 100644 index db5644a..0000000 --- a/models/README.md +++ /dev/null @@ -1 +0,0 @@ -Put pretrained models here diff --git a/python-package/insightface/__init__.py b/python-package/insightface/__init__.py index 571cc5f..760be22 100644 --- a/python-package/insightface/__init__.py +++ b/python-package/insightface/__init__.py @@ -18,11 +18,11 @@ try: except ImportError: raise ImportError( "Unable to import dependency mxnet. " - "A quick tip is to install via `pip install mxnet-mkl/mxnet-cu90mkl --pre`. ") + "A quick tip is to install via `pip install mxnet-mkl/mxnet-cu90mkl --pre`. " + ) __version__ = '0.1.5' from . import model_zoo from . import utils from . import app - diff --git a/python-package/insightface/app/face_analysis.py b/python-package/insightface/app/face_analysis.py index 938b6b3..8df653b 100644 --- a/python-package/insightface/app/face_analysis.py +++ b/python-package/insightface/app/face_analysis.py @@ -7,16 +7,21 @@ import mxnet.ndarray as nd from ..model_zoo import model_zoo from ..utils import face_align -__all__ = ['FaceAnalysis', - 'Face'] +__all__ = ['FaceAnalysis', 'Face'] Face = collections.namedtuple('Face', [ - 'bbox', 'landmark', 'det_score', 'embedding', 'gender', 'age', 'embedding_norm', 'normed_embedding']) + 'bbox', 'landmark', 'det_score', 'embedding', 'gender', 'age', + 'embedding_norm', 'normed_embedding' +]) + +Face.__new__.__defaults__ = (None, ) * len(Face._fields) -Face.__new__.__defaults__ = (None,) * len(Face._fields) class FaceAnalysis: - def __init__(self, det_name='retinaface_r50_v1', rec_name='arcface_r100_v1', ga_name='genderage_v1'): + def __init__(self, + det_name='retinaface_r50_v1', + rec_name='arcface_r100_v1', + ga_name='genderage_v1'): assert det_name is not None self.det_model = model_zoo.get_model(det_name) if rec_name is not None: @@ -35,26 +40,33 @@ class FaceAnalysis: if self.ga_model is not None: self.ga_model.prepare(ctx_id) - def get(self, img, det_thresh = 0.8, det_scale = 1.0, max_num = 0): - bboxes, landmarks = self.det_model.detect(img, threshold=det_thresh, scale = det_scale) - if bboxes.shape[0]==0: + def get(self, img, det_thresh=0.8, det_scale=1.0, max_num=0): + bboxes, landmarks = self.det_model.detect(img, + threshold=det_thresh, + scale=det_scale) + if bboxes.shape[0] == 0: return [] - if max_num>0 and bboxes.shape[0]>max_num: - area = (bboxes[:,2]-bboxes[:,0])*(bboxes[:,3]-bboxes[:,1]) - img_center = img.shape[0]//2, img.shape[1]//2 - offsets = np.vstack([ (bboxes[:,0]+bboxes[:,2])/2-img_center[1], (bboxes[:,1]+bboxes[:,3])/2-img_center[0] ]) - offset_dist_squared = np.sum(np.power(offsets,2.0),0) - values = area-offset_dist_squared*2.0 # some extra weight on the centering - bindex = np.argsort(values)[::-1] # some extra weight on the centering + if max_num > 0 and bboxes.shape[0] > max_num: + area = (bboxes[:, 2] - bboxes[:, 0]) * (bboxes[:, 3] - + bboxes[:, 1]) + img_center = img.shape[0] // 2, img.shape[1] // 2 + offsets = np.vstack([ + (bboxes[:, 0] + bboxes[:, 2]) / 2 - img_center[1], + (bboxes[:, 1] + bboxes[:, 3]) / 2 - img_center[0] + ]) + offset_dist_squared = np.sum(np.power(offsets, 2.0), 0) + values = area - offset_dist_squared * 2.0 # some extra weight on the centering + bindex = np.argsort( + values)[::-1] # some extra weight on the centering bindex = bindex[0:max_num] bboxes = bboxes[bindex, :] landmarks = landmarks[bindex, :] ret = [] for i in range(bboxes.shape[0]): bbox = bboxes[i, 0:4] - det_score = bboxes[i,4] + det_score = bboxes[i, 4] landmark = landmarks[i] - _img = face_align.norm_crop(img, landmark = landmark) + _img = face_align.norm_crop(img, landmark=landmark) embedding = None embedding_norm = None normed_embedding = None @@ -66,8 +78,13 @@ class FaceAnalysis: normed_embedding = embedding / embedding_norm if self.ga_model is not None: gender, age = self.ga_model.get(_img) - face = Face(bbox = bbox, landmark = landmark, det_score = det_score, embedding = embedding, gender = gender, age = age - , normed_embedding=normed_embedding, embedding_norm = embedding_norm) + face = Face(bbox=bbox, + landmark=landmark, + det_score=det_score, + embedding=embedding, + gender=gender, + age=age, + normed_embedding=normed_embedding, + embedding_norm=embedding_norm) ret.append(face) return ret - diff --git a/python-package/insightface/model_zoo/face_detection.py b/python-package/insightface/model_zoo/face_detection.py index f04d665..e94ada2 100644 --- a/python-package/insightface/model_zoo/face_detection.py +++ b/python-package/insightface/model_zoo/face_detection.py @@ -4,11 +4,11 @@ import numpy as np import mxnet.ndarray as nd import cv2 -__all__ = ['FaceDetector', - 'retinaface_r50_v1', - 'retinaface_mnet025_v1', - 'retinaface_mnet025_v2', - 'get_retinaface'] +__all__ = [ + 'FaceDetector', 'retinaface_r50_v1', 'retinaface_mnet025_v1', + 'retinaface_mnet025_v2', 'get_retinaface' +] + def _whctrs(anchor): """ @@ -30,12 +30,11 @@ def _mkanchors(ws, hs, x_ctr, y_ctr): ws = ws[:, np.newaxis] hs = hs[:, np.newaxis] - anchors = np.hstack((x_ctr - 0.5 * (ws - 1), - y_ctr - 0.5 * (hs - 1), - x_ctr + 0.5 * (ws - 1), - y_ctr + 0.5 * (hs - 1))) + anchors = np.hstack((x_ctr - 0.5 * (ws - 1), y_ctr - 0.5 * (hs - 1), + x_ctr + 0.5 * (ws - 1), y_ctr + 0.5 * (hs - 1))) return anchors + def _ratio_enum(anchor, ratios): """ Enumerate a set of anchors for each aspect ratio wrt an anchor. @@ -61,6 +60,7 @@ def _scale_enum(anchor, scales): anchors = _mkanchors(ws, hs, x_ctr, y_ctr) return anchors + def anchors_plane(height, width, stride, base_anchors): """ Parameters @@ -86,8 +86,11 @@ def anchors_plane(height, width, stride, base_anchors): all_anchors[ih, iw, k, 3] = base_anchors[k, 3] + sh return all_anchors -def generate_anchors(base_size=16, ratios=[0.5, 1, 2], - scales=2 ** np.arange(3, 6), stride=16): + +def generate_anchors(base_size=16, + ratios=[0.5, 1, 2], + scales=2**np.arange(3, 6), + stride=16): """ Generate anchor (reference) windows by enumerating aspect ratios X scales wrt a reference (0, 0, 15, 15) window. @@ -95,10 +98,13 @@ def generate_anchors(base_size=16, ratios=[0.5, 1, 2], base_anchor = np.array([1, 1, base_size, base_size]) - 1 ratio_anchors = _ratio_enum(base_anchor, ratios) - anchors = np.vstack([_scale_enum(ratio_anchors[i, :], scales) - for i in range(ratio_anchors.shape[0])]) + anchors = np.vstack([ + _scale_enum(ratio_anchors[i, :], scales) + for i in range(ratio_anchors.shape[0]) + ]) return anchors + def generate_anchors_fpn(cfg): """ Generate anchor (reference) windows by enumerating aspect ratios X @@ -106,22 +112,23 @@ def generate_anchors_fpn(cfg): """ RPN_FEAT_STRIDE = [] for k in cfg: - RPN_FEAT_STRIDE.append( int(k) ) + RPN_FEAT_STRIDE.append(int(k)) RPN_FEAT_STRIDE = sorted(RPN_FEAT_STRIDE, reverse=True) anchors = [] for k in RPN_FEAT_STRIDE: - v = cfg[str(k)] - bs = v['BASE_SIZE'] - __ratios = np.array(v['RATIOS']) - __scales = np.array(v['SCALES']) - stride = int(k) - #print('anchors_fpn', bs, __ratios, __scales, file=sys.stderr) - r = generate_anchors(bs, __ratios, __scales, stride) - #print('anchors_fpn', r.shape, file=sys.stderr) - anchors.append(r) + v = cfg[str(k)] + bs = v['BASE_SIZE'] + __ratios = np.array(v['RATIOS']) + __scales = np.array(v['SCALES']) + stride = int(k) + #print('anchors_fpn', bs, __ratios, __scales, file=sys.stderr) + r = generate_anchors(bs, __ratios, __scales, stride) + #print('anchors_fpn', r.shape, file=sys.stderr) + anchors.append(r) return anchors + def clip_pad(tensor, pad_shape): """ Clip boxes of the pad area. @@ -133,10 +140,11 @@ def clip_pad(tensor, pad_shape): h, w = pad_shape if h < H or w < W: - tensor = tensor[:, :, :h, :w].copy() + tensor = tensor[:, :, :h, :w].copy() return tensor + def bbox_pred(boxes, box_deltas): """ Transform the set of class-agnostic boxes into class-specific boxes @@ -174,11 +182,12 @@ def bbox_pred(boxes, box_deltas): # y2 pred_boxes[:, 3:4] = pred_ctr_y + 0.5 * (pred_h - 1.0) - if box_deltas.shape[1]>4: - pred_boxes[:,4:] = box_deltas[:,4:] + if box_deltas.shape[1] > 4: + pred_boxes[:, 4:] = box_deltas[:, 4:] return pred_boxes + def landmark_pred(boxes, landmark_deltas): if boxes.shape[0] == 0: return np.zeros((0, landmark_deltas.shape[1])) @@ -189,10 +198,11 @@ def landmark_pred(boxes, landmark_deltas): ctr_y = boxes[:, 1] + 0.5 * (heights - 1.0) pred = landmark_deltas.copy() for i in range(5): - pred[:,i,0] = landmark_deltas[:,i,0]*widths + ctr_x - pred[:,i,1] = landmark_deltas[:,i,1]*heights + ctr_y + pred[:, i, 0] = landmark_deltas[:, i, 0] * widths + ctr_x + pred[:, i, 1] = landmark_deltas[:, i, 1] * heights + ctr_y return pred + class FaceDetector: def __init__(self, param_file, rac): self.param_file = param_file @@ -203,55 +213,74 @@ class FaceDetector: pos = self.param_file.rfind('-') prefix = self.param_file[0:pos] pos2 = self.param_file.rfind('.') - epoch = int(self.param_file[pos+1:pos2]) + epoch = int(self.param_file[pos + 1:pos2]) sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) - if ctx_id>=0: + if ctx_id >= 0: ctx = mx.gpu(ctx_id) else: ctx = mx.cpu() - model = mx.mod.Module(symbol=sym, context=ctx, label_names = None) + model = mx.mod.Module(symbol=sym, context=ctx, label_names=None) if fix_image_size is not None: - data_shape = (1,3)+fix_image_size + data_shape = (1, 3) + fix_image_size else: - data_shape = (1,3)+self.default_image_size + data_shape = (1, 3) + self.default_image_size model.bind(data_shapes=[('data', data_shape)]) model.set_params(arg_params, aux_params) #warmup data = mx.nd.zeros(shape=data_shape) - db = mx.io.DataBatch(data=(data,)) + db = mx.io.DataBatch(data=(data, )) model.forward(db, is_train=False) out = model.get_outputs()[0].asnumpy() self.model = model self.nms_threshold = nms self.landmark_std = 1.0 - _ratio = (1.,) + _ratio = (1., ) fmc = 3 - if self.rac=='net3': - _ratio = (1.,) - elif self.rac=='net3l': - _ratio = (1.,) + if self.rac == 'net3': + _ratio = (1., ) + elif self.rac == 'net3l': + _ratio = (1., ) self.landmark_std = 0.2 - elif network=='net5': #retinaface + elif network == 'net5': #retinaface fmc = 5 else: - assert False, 'rac setting error %s'%self.rac + assert False, 'rac setting error %s' % self.rac - if fmc==3: + if fmc == 3: self._feat_stride_fpn = [32, 16, 8] self.anchor_cfg = { - '32': {'SCALES': (32,16), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999}, - '16': {'SCALES': (8,4), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999}, - '8': {'SCALES': (2,1), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999}, - } - elif fmc==5: + '32': { + 'SCALES': (32, 16), + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + }, + '16': { + 'SCALES': (8, 4), + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + }, + '8': { + 'SCALES': (2, 1), + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + }, + } + elif fmc == 5: self._feat_stride_fpn = [64, 32, 16, 8, 4] self.anchor_cfg = {} - _ass = 2.0**(1.0/3) + _ass = 2.0**(1.0 / 3) _basescale = 1.0 for _stride in [4, 8, 16, 32, 64]: key = str(_stride) - value = {'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999} + value = { + 'BASE_SIZE': 16, + 'RATIOS': _ratio, + 'ALLOWED_BORDER': 9999 + } scales = [] for _ in range(3): scales.append(_basescale) @@ -261,61 +290,70 @@ class FaceDetector: print(self._feat_stride_fpn, self.anchor_cfg) self.use_landmarks = False - if len(sym)//len(self._feat_stride_fpn)==3: + if len(sym) // len(self._feat_stride_fpn) == 3: self.use_landmarks = True print('use_landmarks', self.use_landmarks) self.fpn_keys = [] for s in self._feat_stride_fpn: - self.fpn_keys.append('stride%s'%s) + self.fpn_keys.append('stride%s' % s) - self._anchors_fpn = dict(zip(self.fpn_keys, generate_anchors_fpn(cfg=self.anchor_cfg))) + self._anchors_fpn = dict( + zip(self.fpn_keys, generate_anchors_fpn(cfg=self.anchor_cfg))) for k in self._anchors_fpn: v = self._anchors_fpn[k].astype(np.float32) self._anchors_fpn[k] = v self.anchor_plane_cache = {} - self._num_anchors = dict(zip(self.fpn_keys, [anchors.shape[0] for anchors in self._anchors_fpn.values()])) + self._num_anchors = dict( + zip(self.fpn_keys, + [anchors.shape[0] for anchors in self._anchors_fpn.values()])) def detect(self, img, threshold=0.5, scale=1.0): proposals_list = [] scores_list = [] landmarks_list = [] - if scale==1.0: + if scale == 1.0: im = img else: - im = cv2.resize(img, None, None, fx=scale, fy=scale, interpolation=cv2.INTER_LINEAR) + im = cv2.resize(img, + None, + None, + fx=scale, + fy=scale, + interpolation=cv2.INTER_LINEAR) im_info = [im.shape[0], im.shape[1]] im_tensor = np.zeros((1, 3, im.shape[0], im.shape[1])) for i in range(3): im_tensor[0, i, :, :] = im[:, :, 2 - i] data = nd.array(im_tensor) - db = mx.io.DataBatch(data=(data,), provide_data=[('data', data.shape)]) + db = mx.io.DataBatch(data=(data, ), + provide_data=[('data', data.shape)]) self.model.forward(db, is_train=False) net_out = self.model.get_outputs() - for _idx,s in enumerate(self._feat_stride_fpn): - _key = 'stride%s'%s + for _idx, s in enumerate(self._feat_stride_fpn): + _key = 'stride%s' % s stride = int(s) if self.use_landmarks: - idx = _idx*3 + idx = _idx * 3 else: - idx = _idx*2 + idx = _idx * 2 scores = net_out[idx].asnumpy() - scores = scores[:, self._num_anchors['stride%s'%s]:, :, :] - idx+=1 + scores = scores[:, self._num_anchors['stride%s' % s]:, :, :] + idx += 1 bbox_deltas = net_out[idx].asnumpy() height, width = bbox_deltas.shape[2], bbox_deltas.shape[3] - A = self._num_anchors['stride%s'%s] + A = self._num_anchors['stride%s' % s] K = height * width key = (height, width, stride) if key in self.anchor_plane_cache: anchors = self.anchor_plane_cache[key] else: - anchors_fpn = self._anchors_fpn['stride%s'%s] + anchors_fpn = self._anchors_fpn['stride%s' % s] anchors = anchors_plane(height, width, stride, anchors_fpn) anchors = anchors.reshape((K * A, 4)) - if len(self.anchor_plane_cache)<100: + if len(self.anchor_plane_cache) < 100: self.anchor_plane_cache[key] = anchors scores = clip_pad(scores, (height, width)) @@ -323,43 +361,43 @@ class FaceDetector: bbox_deltas = clip_pad(bbox_deltas, (height, width)) bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)) - bbox_pred_len = bbox_deltas.shape[3]//A + bbox_pred_len = bbox_deltas.shape[3] // A bbox_deltas = bbox_deltas.reshape((-1, bbox_pred_len)) proposals = bbox_pred(anchors, bbox_deltas) #proposals = clip_boxes(proposals, im_info[:2]) - scores_ravel = scores.ravel() - order = np.where(scores_ravel>=threshold)[0] + order = np.where(scores_ravel >= threshold)[0] proposals = proposals[order, :] scores = scores[order] - proposals[:,0:4] /= scale + proposals[:, 0:4] /= scale proposals_list.append(proposals) scores_list.append(scores) if self.use_landmarks: - idx+=1 + idx += 1 landmark_deltas = net_out[idx].asnumpy() landmark_deltas = clip_pad(landmark_deltas, (height, width)) - landmark_pred_len = landmark_deltas.shape[1]//A - landmark_deltas = landmark_deltas.transpose((0, 2, 3, 1)).reshape((-1, 5, landmark_pred_len//5)) + landmark_pred_len = landmark_deltas.shape[1] // A + landmark_deltas = landmark_deltas.transpose( + (0, 2, 3, 1)).reshape((-1, 5, landmark_pred_len // 5)) landmark_deltas *= self.landmark_std #print(landmark_deltas.shape, landmark_deltas) landmarks = landmark_pred(anchors, landmark_deltas) landmarks = landmarks[order, :] - landmarks[:,:,0:2] /= scale + landmarks[:, :, 0:2] /= scale landmarks_list.append(landmarks) proposals = np.vstack(proposals_list) landmarks = None - if proposals.shape[0]==0: + if proposals.shape[0] == 0: if self.use_landmarks: - landmarks = np.zeros( (0,5,2) ) - return np.zeros( (0,5) ), landmarks + landmarks = np.zeros((0, 5, 2)) + return np.zeros((0, 5)), landmarks scores = np.vstack(scores_list) scores_ravel = scores.ravel() order = scores_ravel.argsort()[::-1] @@ -369,9 +407,10 @@ class FaceDetector: landmarks = np.vstack(landmarks_list) landmarks = landmarks[order].astype(np.float32, copy=False) - pre_det = np.hstack((proposals[:,0:4], scores)).astype(np.float32, copy=False) + pre_det = np.hstack((proposals[:, 0:4], scores)).astype(np.float32, + copy=False) keep = self.nms(pre_det) - det = np.hstack( (pre_det, proposals[:,4:]) ) + det = np.hstack((pre_det, proposals[:, 4:])) det = det[keep, :] if self.use_landmarks: landmarks = landmarks[keep] @@ -409,18 +448,19 @@ class FaceDetector: return keep -def get_retinaface(name, rac='net3', - root='~/.insightface/models', **kwargs): +def get_retinaface(name, rac='net3', root='~/.insightface/models', **kwargs): from .model_store import get_model_file - _file = get_model_file("retinaface_%s"%name, root=root) + _file = get_model_file("retinaface_%s" % name, root=root) return FaceDetector(_file, rac) + def retinaface_r50_v1(**kwargs): return get_retinaface("r50_v1", rac='net3', **kwargs) + def retinaface_mnet025_v1(**kwargs): return get_retinaface("mnet025_v1", rac='net3', **kwargs) + def retinaface_mnet025_v2(**kwargs): return get_retinaface("mnet025_v2", rac='net3l', **kwargs) - diff --git a/python-package/insightface/model_zoo/face_genderage.py b/python-package/insightface/model_zoo/face_genderage.py index 3a0423d..69f56e1 100644 --- a/python-package/insightface/model_zoo/face_genderage.py +++ b/python-package/insightface/model_zoo/face_genderage.py @@ -3,9 +3,7 @@ import mxnet as mx import numpy as np import cv2 -__all__ = ['FaceGenderage', - 'genderage_v1', - 'get_genderage'] +__all__ = ['FaceGenderage', 'genderage_v1', 'get_genderage'] class FaceGenderage: @@ -22,21 +20,22 @@ class FaceGenderage: pos = self.param_file.rfind('-') prefix = self.param_file[0:pos] pos2 = self.param_file.rfind('.') - epoch = int(self.param_file[pos+1:pos2]) - sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) + epoch = int(self.param_file[pos + 1:pos2]) + sym, arg_params, aux_params = mx.model.load_checkpoint( + prefix, epoch) all_layers = sym.get_internals() sym = all_layers['fc1_output'] - if ctx_id>=0: + if ctx_id >= 0: ctx = mx.gpu(ctx_id) else: ctx = mx.cpu() - model = mx.mod.Module(symbol=sym, context=ctx, label_names = None) - data_shape = (1,3)+self.image_size + model = mx.mod.Module(symbol=sym, context=ctx, label_names=None) + data_shape = (1, 3) + self.image_size model.bind(data_shapes=[('data', data_shape)]) model.set_params(arg_params, aux_params) #warmup data = mx.nd.zeros(shape=data_shape) - db = mx.io.DataBatch(data=(data,)) + db = mx.io.DataBatch(data=(data, )) model.forward(db, is_train=False) embedding = model.get_outputs()[0].asnumpy() self.model = model @@ -45,33 +44,30 @@ class FaceGenderage: def get(self, img): assert self.param_file and self.model - assert img.shape[2]==3 and img.shape[0:2]==self.image_size + assert img.shape[2] == 3 and img.shape[0:2] == self.image_size data = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) - data = np.transpose(data, (2,0,1)) + data = np.transpose(data, (2, 0, 1)) data = np.expand_dims(data, axis=0) data = mx.nd.array(data) - db = mx.io.DataBatch(data=(data,)) + db = mx.io.DataBatch(data=(data, )) self.model.forward(db, is_train=False) ret = self.model.get_outputs()[0].asnumpy() - g = ret[:,0:2].flatten() + g = ret[:, 0:2].flatten() gender = np.argmax(g) - a = ret[:,2:202].reshape( (100,2) ) + a = ret[:, 2:202].reshape((100, 2)) a = np.argmax(a, axis=1) age = int(sum(a)) return gender, age -def get_genderage(name, download=True, - root='~/.insightface/models', **kwargs): + +def get_genderage(name, download=True, root='~/.insightface/models', **kwargs): if not download: return FaceGenderage(name, False, None) else: from .model_store import get_model_file - _file = get_model_file("genderage_%s"%name, root=root) + _file = get_model_file("genderage_%s" % name, root=root) return FaceGenderage(name, True, _file) + def genderage_v1(**kwargs): return get_genderage("v1", download=True, **kwargs) - - - - diff --git a/python-package/insightface/model_zoo/face_recognition.py b/python-package/insightface/model_zoo/face_recognition.py index 506ff1d..e8a5f92 100644 --- a/python-package/insightface/model_zoo/face_recognition.py +++ b/python-package/insightface/model_zoo/face_recognition.py @@ -3,9 +3,10 @@ import mxnet as mx import numpy as np import cv2 -__all__ = ['FaceRecognition', - 'arcface_r100_v1', 'arcface_outofreach_v1', 'arcface_mfn_v1', - 'get_arcface'] +__all__ = [ + 'FaceRecognition', 'arcface_r100_v1', 'arcface_outofreach_v1', + 'arcface_mfn_v1', 'get_arcface' +] class FaceRecognition: @@ -22,21 +23,22 @@ class FaceRecognition: pos = self.param_file.rfind('-') prefix = self.param_file[0:pos] pos2 = self.param_file.rfind('.') - epoch = int(self.param_file[pos+1:pos2]) - sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) + epoch = int(self.param_file[pos + 1:pos2]) + sym, arg_params, aux_params = mx.model.load_checkpoint( + prefix, epoch) all_layers = sym.get_internals() sym = all_layers['fc1_output'] - if ctx_id>=0: + if ctx_id >= 0: ctx = mx.gpu(ctx_id) else: ctx = mx.cpu() - model = mx.mod.Module(symbol=sym, context=ctx, label_names = None) - data_shape = (1,3)+self.image_size + model = mx.mod.Module(symbol=sym, context=ctx, label_names=None) + data_shape = (1, 3) + self.image_size model.bind(data_shapes=[('data', data_shape)]) model.set_params(arg_params, aux_params) #warmup data = mx.nd.zeros(shape=data_shape) - db = mx.io.DataBatch(data=(data,)) + db = mx.io.DataBatch(data=(data, )) model.forward(db, is_train=False) embedding = model.get_outputs()[0].asnumpy() self.model = model @@ -45,12 +47,12 @@ class FaceRecognition: def get_embedding(self, img): assert self.param_file and self.model - assert img.shape[2]==3 and img.shape[0:2]==self.image_size + assert img.shape[2] == 3 and img.shape[0:2] == self.image_size data = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) - data = np.transpose(data, (2,0,1)) + data = np.transpose(data, (2, 0, 1)) data = np.expand_dims(data, axis=0) data = mx.nd.array(data) - db = mx.io.DataBatch(data=(data,)) + db = mx.io.DataBatch(data=(data, )) self.model.forward(db, is_train=False) embedding = self.model.get_outputs()[0].asnumpy() return embedding @@ -59,18 +61,19 @@ class FaceRecognition: emb1 = self.get_embedding(img1).flatten() emb2 = self.get_embedding(img2).flatten() from numpy.linalg import norm - sim = np.dot(emb1, emb2)/(norm(emb1)*norm(emb2)) + sim = np.dot(emb1, emb2) / (norm(emb1) * norm(emb2)) return sim -def get_arcface(name, download=True, - root='~/.insightface/models', **kwargs): + +def get_arcface(name, download=True, root='~/.insightface/models', **kwargs): if not download: return FaceRecognition(name, False, None) else: from .model_store import get_model_file - _file = get_model_file("arcface_%s"%name, root=root) + _file = get_model_file("arcface_%s" % name, root=root) return FaceRecognition(name, True, _file) + def arcface_r100_v1(**kwargs): return get_arcface("r100_v1", download=True, **kwargs) @@ -78,6 +81,6 @@ def arcface_r100_v1(**kwargs): def arcface_mfn_v1(**kwargs): return get_arcface("mfn_v1", download=True, **kwargs) + def arcface_outofreach_v1(**kwargs): return get_arcface("outofreach_v1", download=False, **kwargs) - diff --git a/python-package/insightface/model_zoo/model_store.py b/python-package/insightface/model_zoo/model_store.py index e76eafb..59c6945 100644 --- a/python-package/insightface/model_zoo/model_store.py +++ b/python-package/insightface/model_zoo/model_store.py @@ -1,4 +1,3 @@ - """ This code file mainly comes from https://github.com/dmlc/gluon-cv/blob/master/gluoncv/model_zoo/model_store.py """ @@ -11,14 +10,17 @@ import glob from ..utils import download, check_sha1 -_model_sha1 = {name: checksum for checksum, name in [ - ('95be21b58e29e9c1237f229dae534bd854009ce0', 'arcface_r100_v1'), - ('', 'arcface_mfn_v1'), - ('39fd1e087a2a2ed70a154ac01fecaa86c315d01b', 'retinaface_r50_v1'), - ('2c9de8116d1f448fd1d4661f90308faae34c990a', 'retinaface_mnet025_v1'), - ('0db1d07921d005e6c9a5b38e059452fc5645e5a4', 'retinaface_mnet025_v2'), - ('7dd8111652b7aac2490c5dcddeb268e53ac643e6', 'genderage_v1'), -]} +_model_sha1 = { + name: checksum + for checksum, name in [ + ('95be21b58e29e9c1237f229dae534bd854009ce0', 'arcface_r100_v1'), + ('', 'arcface_mfn_v1'), + ('39fd1e087a2a2ed70a154ac01fecaa86c315d01b', 'retinaface_r50_v1'), + ('2c9de8116d1f448fd1d4661f90308faae34c990a', 'retinaface_mnet025_v1'), + ('0db1d07921d005e6c9a5b38e059452fc5645e5a4', 'retinaface_mnet025_v2'), + ('7dd8111652b7aac2490c5dcddeb268e53ac643e6', 'genderage_v1'), + ] +} base_repo_url = 'http://insightface.ai/files/' _url_format = '{repo_url}models/{file_name}.zip' @@ -26,19 +28,21 @@ _url_format = '{repo_url}models/{file_name}.zip' def short_hash(name): if name not in _model_sha1: - raise ValueError('Pretrained model for {name} is not available.'.format(name=name)) + raise ValueError( + 'Pretrained model for {name} is not available.'.format(name=name)) return _model_sha1[name][:8] def find_params_file(dir_path): if not os.path.exists(dir_path): return None - paths = glob.glob("%s/*.params"%dir_path) - if len(paths)==0: + paths = glob.glob("%s/*.params" % dir_path) + if len(paths) == 0: return None paths = sorted(paths) return paths[-1] + def get_model_file(name, root=os.path.join('~', '.insightface', 'models')): r"""Return location for the pretrained on local file system. @@ -68,7 +72,9 @@ def get_model_file(name, root=os.path.join('~', '.insightface', 'models')): if check_sha1(file_path, sha1_hash): return file_path else: - print('Mismatch in the content of model file detected. Downloading again.') + print( + 'Mismatch in the content of model file detected. Downloading again.' + ) else: print('Model file is not found. Downloading.') @@ -92,6 +98,5 @@ def get_model_file(name, root=os.path.join('~', '.insightface', 'models')): if check_sha1(file_path, sha1_hash): return file_path else: - raise ValueError('Downloaded file has different hash. Please try again.') - - + raise ValueError( + 'Downloaded file has different hash. Please try again.') diff --git a/python-package/insightface/model_zoo/model_zoo.py b/python-package/insightface/model_zoo/model_zoo.py index c3298ad..f394a6a 100644 --- a/python-package/insightface/model_zoo/model_zoo.py +++ b/python-package/insightface/model_zoo/model_zoo.py @@ -54,4 +54,3 @@ def get_model_list(): """ return sorted(_models.keys()) - diff --git a/python-package/insightface/utils/download.py b/python-package/insightface/utils/download.py index 8f96fe6..d7b4051 100644 --- a/python-package/insightface/utils/download.py +++ b/python-package/insightface/utils/download.py @@ -6,6 +6,7 @@ import hashlib import requests from tqdm import tqdm + def check_sha1(filename, sha1_hash): """Check whether the sha1 hash of the file content matches the expected hash. Parameters @@ -31,6 +32,7 @@ def check_sha1(filename, sha1_hash): l = min(len(sha1_file), len(sha1_hash)) return sha1.hexdigest()[0:l] == sha1_hash[0:l] + def download(url, path=None, overwrite=False, sha1_hash=None): """Download an given URL Parameters @@ -59,26 +61,29 @@ def download(url, path=None, overwrite=False, sha1_hash=None): else: fname = path - if overwrite or not os.path.exists(fname) or (sha1_hash and not check_sha1(fname, sha1_hash)): + if overwrite or not os.path.exists(fname) or ( + sha1_hash and not check_sha1(fname, sha1_hash)): dirname = os.path.dirname(os.path.abspath(os.path.expanduser(fname))) if not os.path.exists(dirname): os.makedirs(dirname) - print('Downloading %s from %s...'%(fname, url)) + print('Downloading %s from %s...' % (fname, url)) r = requests.get(url, stream=True) if r.status_code != 200: - raise RuntimeError("Failed downloading url %s"%url) + raise RuntimeError("Failed downloading url %s" % url) total_length = r.headers.get('content-length') with open(fname, 'wb') as f: - if total_length is None: # no content length header + if total_length is None: # no content length header for chunk in r.iter_content(chunk_size=1024): - if chunk: # filter out keep-alive new chunks + if chunk: # filter out keep-alive new chunks f.write(chunk) else: total_length = int(total_length) for chunk in tqdm(r.iter_content(chunk_size=1024), total=int(total_length / 1024. + 0.5), - unit='KB', unit_scale=False, dynamic_ncols=True): + unit='KB', + unit_scale=False, + dynamic_ncols=True): f.write(chunk) if sha1_hash and not check_sha1(fname, sha1_hash): diff --git a/python-package/insightface/utils/face_align.py b/python-package/insightface/utils/face_align.py index 6cf5a2e..4f48a76 100644 --- a/python-package/insightface/utils/face_align.py +++ b/python-package/insightface/utils/face_align.py @@ -1,88 +1,71 @@ - import cv2 import numpy as np from skimage import transform as trans -src1 = np.array([ - [51.642,50.115], - [57.617,49.990], - [35.740,69.007], - [51.157,89.050], - [57.025,89.702]], dtype=np.float32) -#<--left -src2 = np.array([ - [45.031,50.118], - [65.568,50.872], - [39.677,68.111], - [45.177,86.190], - [64.246,86.758]], dtype=np.float32) +src1 = np.array([[51.642, 50.115], [57.617, 49.990], [35.740, 69.007], + [51.157, 89.050], [57.025, 89.702]], + dtype=np.float32) +#<--left +src2 = np.array([[45.031, 50.118], [65.568, 50.872], [39.677, 68.111], + [45.177, 86.190], [64.246, 86.758]], + dtype=np.float32) #---frontal -src3 = np.array([ - [39.730,51.138], - [72.270,51.138], - [56.000,68.493], - [42.463,87.010], - [69.537,87.010]], dtype=np.float32) +src3 = np.array([[39.730, 51.138], [72.270, 51.138], [56.000, 68.493], + [42.463, 87.010], [69.537, 87.010]], + dtype=np.float32) #-->right -src4 = np.array([ - [46.845,50.872], - [67.382,50.118], - [72.737,68.111], - [48.167,86.758], - [67.236,86.190]], dtype=np.float32) +src4 = np.array([[46.845, 50.872], [67.382, 50.118], [72.737, 68.111], + [48.167, 86.758], [67.236, 86.190]], + dtype=np.float32) #-->right profile -src5 = np.array([ - [54.796,49.990], - [60.771,50.115], - [76.673,69.007], - [55.388,89.702], - [61.257,89.050]], dtype=np.float32) +src5 = np.array([[54.796, 49.990], [60.771, 50.115], [76.673, 69.007], + [55.388, 89.702], [61.257, 89.050]], + dtype=np.float32) -src = np.array([src1,src2,src3,src4,src5]) -src_map = {112 : src, 224 : src*2} +src = np.array([src1, src2, src3, src4, src5]) +src_map = {112: src, 224: src * 2} -arcface_src = np.array([ - [38.2946, 51.6963], - [73.5318, 51.5014], - [56.0252, 71.7366], - [41.5493, 92.3655], - [70.7299, 92.2041] ], dtype=np.float32 ) +arcface_src = np.array( + [[38.2946, 51.6963], [73.5318, 51.5014], [56.0252, 71.7366], + [41.5493, 92.3655], [70.7299, 92.2041]], + dtype=np.float32) arcface_src = np.expand_dims(arcface_src, axis=0) # In[66]: + # lmk is prediction; src is template -def estimate_norm(lmk, image_size = 112, mode='arcface'): - assert lmk.shape==(5,2) - tform = trans.SimilarityTransform() - lmk_tran = np.insert(lmk, 2, values=np.ones(5), axis=1) - min_M = [] - min_index = [] - min_error = float('inf') - if mode=='arcface': - assert image_size==112 - src = arcface_src - else: - src = src_map[image_size] - for i in np.arange(src.shape[0]): - tform.estimate(lmk, src[i]) - M = tform.params[0:2,:] - results = np.dot(M, lmk_tran.T) - results = results.T - error = np.sum(np.sqrt(np.sum((results - src[i]) ** 2,axis=1))) -# print(error) - if error< min_error: - min_error = error - min_M = M - min_index = i - return min_M, min_index +def estimate_norm(lmk, image_size=112, mode='arcface'): + assert lmk.shape == (5, 2) + tform = trans.SimilarityTransform() + lmk_tran = np.insert(lmk, 2, values=np.ones(5), axis=1) + min_M = [] + min_index = [] + min_error = float('inf') + if mode == 'arcface': + assert image_size == 112 + src = arcface_src + else: + src = src_map[image_size] + for i in np.arange(src.shape[0]): + tform.estimate(lmk, src[i]) + M = tform.params[0:2, :] + results = np.dot(M, lmk_tran.T) + results = results.T + error = np.sum(np.sqrt(np.sum((results - src[i])**2, axis=1))) + # print(error) + if error < min_error: + min_error = error + min_M = M + min_index = i + return min_M, min_index + def norm_crop(img, landmark, image_size=112, mode='arcface'): - M, pose_index = estimate_norm(landmark, image_size, mode) - warped = cv2.warpAffine(img,M, (image_size, image_size), borderValue = 0.0) - return warped - + M, pose_index = estimate_norm(landmark, image_size, mode) + warped = cv2.warpAffine(img, M, (image_size, image_size), borderValue=0.0) + return warped diff --git a/python-package/insightface/utils/filesystem.py b/python-package/insightface/utils/filesystem.py index 4ecb091..158f5ed 100644 --- a/python-package/insightface/utils/filesystem.py +++ b/python-package/insightface/utils/filesystem.py @@ -4,6 +4,7 @@ This code file mainly comes from https://github.com/dmlc/gluon-cv/blob/master/gl import os import errno + def makedirs(path): """Create directory recursively if not exists. Similar to `makedir -p`, you can skip checking existence before this function. @@ -19,6 +20,7 @@ def makedirs(path): if exc.errno != errno.EEXIST: raise + def try_import(package, message=None): """Try import specified package, with custom message support. @@ -42,6 +44,7 @@ def try_import(package, message=None): raise e raise ImportError(message) + def try_import_cv2(): """Try import cv2 at runtime. @@ -52,8 +55,10 @@ def try_import_cv2(): """ msg = "cv2 is required, you can install by package manager, e.g. 'apt-get', \ or `pip install opencv-python --user` (note that this is unofficial PYPI package)." + return try_import('cv2', msg) + def try_import_mmcv(): """Try import mmcv at runtime. @@ -64,8 +69,10 @@ def try_import_mmcv(): """ msg = "mmcv is required, you can install by first `pip install Cython --user` \ and then `pip install mmcv --user` (note that this is unofficial PYPI package)." + return try_import('mmcv', msg) + def try_import_rarfile(): """Try import rarfile at runtime. @@ -76,8 +83,10 @@ def try_import_rarfile(): """ msg = "rarfile is required, you can install by first `sudo apt-get install unrar` \ and then `pip install rarfile --user` (note that this is unofficial PYPI package)." + return try_import('rarfile', msg) + def import_try_install(package, extern_url=None): """Try import the specified package. If the package not installed, try use pip to install and import if success. @@ -108,7 +117,8 @@ def import_try_install(package, extern_url=None): # trying to install package url = package if extern_url is None else extern_url - pipmain(['install', '--user', url]) # will raise SystemExit Error if fails + pipmain(['install', '--user', + url]) # will raise SystemExit Error if fails # trying to load again try: @@ -122,6 +132,7 @@ def import_try_install(package, extern_url=None): return __import__(package) return __import__(package) + def try_import_dali(): """Try import NVIDIA DALI at runtime. """ @@ -129,9 +140,12 @@ def try_import_dali(): dali = __import__('nvidia.dali', fromlist=['pipeline', 'ops', 'types']) dali.Pipeline = dali.pipeline.Pipeline except ImportError: + class dali: class Pipeline: def __init__(self): raise NotImplementedError( - "DALI not found, please check if you installed it correctly.") + "DALI not found, please check if you installed it correctly." + ) + return dali diff --git a/python-package/setup.py b/python-package/setup.py index c48bb29..766b975 100644 --- a/python-package/setup.py +++ b/python-package/setup.py @@ -6,11 +6,10 @@ import shutil import sys from setuptools import setup, find_packages + def read(*names, **kwargs): - with io.open( - os.path.join(os.path.dirname(__file__), *names), - encoding=kwargs.get("encoding", "utf8") - ) as fp: + with io.open(os.path.join(os.path.dirname(__file__), *names), + encoding=kwargs.get("encoding", "utf8")) as fp: return fp.read() @@ -22,10 +21,11 @@ def find_version(*file_paths): return version_match.group(1) raise RuntimeError("Unable to find version string.") + try: import pypandoc long_description = pypandoc.convert('README.md', 'rst') -except(IOError, ImportError): +except (IOError, ImportError): long_description = open('README.md').read() VERSION = find_version('insightface', '__init__.py') @@ -58,4 +58,3 @@ setup( include_package_data=True, install_requires=requirements, ) - diff --git a/recognition/ArcFace/image_iter.py b/recognition/ArcFace/image_iter.py index a7ef137..1c1b7cf 100644 --- a/recognition/ArcFace/image_iter.py +++ b/recognition/ArcFace/image_iter.py @@ -22,104 +22,111 @@ logger = logging.getLogger() class FaceImageIter(io.DataIter): - - def __init__(self, batch_size, data_shape, - path_imgrec = None, - shuffle=False, aug_list=None, mean = None, - rand_mirror = False, cutoff = 0, color_jittering = 0, - images_filter = 0, - data_name='data', label_name='softmax_label', **kwargs): + def __init__(self, + batch_size, + data_shape, + path_imgrec=None, + shuffle=False, + aug_list=None, + mean=None, + rand_mirror=False, + cutoff=0, + color_jittering=0, + images_filter=0, + data_name='data', + label_name='softmax_label', + **kwargs): super(FaceImageIter, self).__init__() assert path_imgrec if path_imgrec: - logging.info('loading recordio %s...', - path_imgrec) - path_imgidx = path_imgrec[0:-4]+".idx" - self.imgrec = recordio.MXIndexedRecordIO(path_imgidx, path_imgrec, 'r') # pylint: disable=redefined-variable-type + logging.info('loading recordio %s...', path_imgrec) + path_imgidx = path_imgrec[0:-4] + ".idx" + self.imgrec = recordio.MXIndexedRecordIO(path_imgidx, path_imgrec, + 'r') # pylint: disable=redefined-variable-type s = self.imgrec.read_idx(0) header, _ = recordio.unpack(s) - if header.flag>0: - print('header0 label', header.label) - self.header0 = (int(header.label[0]), int(header.label[1])) - #assert(header.flag==1) - #self.imgidx = range(1, int(header.label[0])) - self.imgidx = [] - self.id2range = {} - self.seq_identity = range(int(header.label[0]), int(header.label[1])) - for identity in self.seq_identity: - s = self.imgrec.read_idx(identity) - header, _ = recordio.unpack(s) - a,b = int(header.label[0]), int(header.label[1]) - count = b-a - if count 0: + print('header0 label', header.label) + self.header0 = (int(header.label[0]), int(header.label[1])) + #assert(header.flag==1) + #self.imgidx = range(1, int(header.label[0])) + self.imgidx = [] + self.id2range = {} + self.seq_identity = range(int(header.label[0]), + int(header.label[1])) + for identity in self.seq_identity: + s = self.imgrec.read_idx(identity) + header, _ = recordio.unpack(s) + a, b = int(header.label[0]), int(header.label[1]) + count = b - a + if count < images_filter: + continue + self.id2range[identity] = (a, b) + self.imgidx += range(a, b) + print('id2range', len(self.id2range)) else: - self.imgidx = list(self.imgrec.keys) + self.imgidx = list(self.imgrec.keys) if shuffle: - self.seq = self.imgidx - self.oseq = self.imgidx - print(len(self.seq)) + self.seq = self.imgidx + self.oseq = self.imgidx + print(len(self.seq)) else: - self.seq = None + self.seq = None self.mean = mean self.nd_mean = None if self.mean: - self.mean = np.array(self.mean, dtype=np.float32).reshape(1,1,3) - self.nd_mean = mx.nd.array(self.mean).reshape((1,1,3)) + self.mean = np.array(self.mean, dtype=np.float32).reshape(1, 1, 3) + self.nd_mean = mx.nd.array(self.mean).reshape((1, 1, 3)) self.check_data_shape(data_shape) - self.provide_data = [(data_name, (batch_size,) + data_shape)] + self.provide_data = [(data_name, (batch_size, ) + data_shape)] self.batch_size = batch_size self.data_shape = data_shape self.shuffle = shuffle - self.image_size = '%d,%d'%(data_shape[1],data_shape[2]) + self.image_size = '%d,%d' % (data_shape[1], data_shape[2]) self.rand_mirror = rand_mirror print('rand_mirror', rand_mirror) self.cutoff = cutoff self.color_jittering = color_jittering self.CJA = mx.image.ColorJitterAug(0.125, 0.125, 0.125) - self.provide_label = [(label_name, (batch_size,))] + self.provide_label = [(label_name, (batch_size, ))] #print(self.provide_label[0][1]) self.cur = 0 self.nbatch = 0 self.is_init = False - def reset(self): """Resets the iterator to the beginning of the data.""" print('call reset()') self.cur = 0 if self.shuffle: - random.shuffle(self.seq) + random.shuffle(self.seq) if self.seq is None and self.imgrec is not None: self.imgrec.reset() def num_samples(self): - return len(self.seq) + return len(self.seq) def next_sample(self): """Helper function for reading in next sample.""" #set total batch size, for example, 1800, and maximum size for each people, for example 45 if self.seq is not None: - while True: - if self.cur >= len(self.seq): - raise StopIteration - idx = self.seq[self.cur] - self.cur += 1 - if self.imgrec is not None: - s = self.imgrec.read_idx(idx) - header, img = recordio.unpack(s) - label = header.label - if not isinstance(label, numbers.Number): - label = label[0] - return label, img, None, None - else: - label, fname, bbox, landmark = self.imglist[idx] - return label, self.read_image(fname), bbox, landmark + while True: + if self.cur >= len(self.seq): + raise StopIteration + idx = self.seq[self.cur] + self.cur += 1 + if self.imgrec is not None: + s = self.imgrec.read_idx(idx) + header, img = recordio.unpack(s) + label = header.label + if not isinstance(label, numbers.Number): + label = label[0] + return label, img, None, None + else: + label, fname, bbox, landmark = self.imglist[idx] + return label, self.read_image(fname), bbox, landmark else: s = self.imgrec.read() if s is None: @@ -128,107 +135,106 @@ class FaceImageIter(io.DataIter): return header.label, img, None, None def brightness_aug(self, src, x): - alpha = 1.0 + random.uniform(-x, x) - src *= alpha - return src + alpha = 1.0 + random.uniform(-x, x) + src *= alpha + return src def contrast_aug(self, src, x): - alpha = 1.0 + random.uniform(-x, x) - coef = nd.array([[[0.299, 0.587, 0.114]]]) - gray = src * coef - gray = (3.0 * (1.0 - alpha) / gray.size) * nd.sum(gray) - src *= alpha - src += gray - return src + alpha = 1.0 + random.uniform(-x, x) + coef = nd.array([[[0.299, 0.587, 0.114]]]) + gray = src * coef + gray = (3.0 * (1.0 - alpha) / gray.size) * nd.sum(gray) + src *= alpha + src += gray + return src def saturation_aug(self, src, x): - alpha = 1.0 + random.uniform(-x, x) - coef = nd.array([[[0.299, 0.587, 0.114]]]) - gray = src * coef - gray = nd.sum(gray, axis=2, keepdims=True) - gray *= (1.0 - alpha) - src *= alpha - src += gray - return src + alpha = 1.0 + random.uniform(-x, x) + coef = nd.array([[[0.299, 0.587, 0.114]]]) + gray = src * coef + gray = nd.sum(gray, axis=2, keepdims=True) + gray *= (1.0 - alpha) + src *= alpha + src += gray + return src def color_aug(self, img, x): - #augs = [self.brightness_aug, self.contrast_aug, self.saturation_aug] - #random.shuffle(augs) - #for aug in augs: - # #print(img.shape) - # img = aug(img, x) - # #print(img.shape) - #return img - return self.CJA(img) + #augs = [self.brightness_aug, self.contrast_aug, self.saturation_aug] + #random.shuffle(augs) + #for aug in augs: + # #print(img.shape) + # img = aug(img, x) + # #print(img.shape) + #return img + return self.CJA(img) def mirror_aug(self, img): - _rd = random.randint(0,1) - if _rd==1: - for c in range(img.shape[2]): - img[:,:,c] = np.fliplr(img[:,:,c]) - return img + _rd = random.randint(0, 1) + if _rd == 1: + for c in range(img.shape[2]): + img[:, :, c] = np.fliplr(img[:, :, c]) + return img def compress_aug(self, img): - from PIL import Image - from io import BytesIO - buf = BytesIO() - img = Image.fromarray(img.asnumpy(), 'RGB') - q = random.randint(2, 20) - img.save(buf, format='JPEG', quality=q) - buf = buf.getvalue() - img = Image.open(BytesIO(buf)) - return nd.array(np.asarray(img, 'float32')) - + from PIL import Image + from io import BytesIO + buf = BytesIO() + img = Image.fromarray(img.asnumpy(), 'RGB') + q = random.randint(2, 20) + img.save(buf, format='JPEG', quality=q) + buf = buf.getvalue() + img = Image.open(BytesIO(buf)) + return nd.array(np.asarray(img, 'float32')) def next(self): if not self.is_init: - self.reset() - self.is_init = True + self.reset() + self.is_init = True """Returns the next batch of data.""" #print('in next', self.cur, self.labelcur) - self.nbatch+=1 + self.nbatch += 1 batch_size = self.batch_size c, h, w = self.data_shape batch_data = nd.empty((batch_size, c, h, w)) if self.provide_label is not None: - batch_label = nd.empty(self.provide_label[0][1]) + batch_label = nd.empty(self.provide_label[0][1]) i = 0 try: while i < batch_size: label, s, bbox, landmark = self.next_sample() _data = self.imdecode(s) - if _data.shape[0]!=self.data_shape[1]: - _data = mx.image.resize_short(_data, self.data_shape[1]) + if _data.shape[0] != self.data_shape[1]: + _data = mx.image.resize_short(_data, self.data_shape[1]) if self.rand_mirror: - _rd = random.randint(0,1) - if _rd==1: - _data = mx.ndarray.flip(data=_data, axis=1) - if self.color_jittering>0: - if self.color_jittering>1: - _rd = random.randint(0,1) - if _rd==1: - _data = self.compress_aug(_data) - #print('do color aug') - _data = _data.astype('float32', copy=False) - #print(_data.__class__) - _data = self.color_aug(_data, 0.125) + _rd = random.randint(0, 1) + if _rd == 1: + _data = mx.ndarray.flip(data=_data, axis=1) + if self.color_jittering > 0: + if self.color_jittering > 1: + _rd = random.randint(0, 1) + if _rd == 1: + _data = self.compress_aug(_data) + #print('do color aug') + _data = _data.astype('float32', copy=False) + #print(_data.__class__) + _data = self.color_aug(_data, 0.125) if self.nd_mean is not None: - _data = _data.astype('float32', copy=False) - _data -= self.nd_mean - _data *= 0.0078125 - if self.cutoff>0: - _rd = random.randint(0,1) - if _rd==1: - #print('do cutoff aug', self.cutoff) - centerh = random.randint(0, _data.shape[0]-1) - centerw = random.randint(0, _data.shape[1]-1) - half = self.cutoff//2 - starth = max(0, centerh-half) - endh = min(_data.shape[0], centerh+half) - startw = max(0, centerw-half) - endw = min(_data.shape[1], centerw+half) - #print(starth, endh, startw, endw, _data.shape) - _data[starth:endh, startw:endw, :] = 128 + _data = _data.astype('float32', copy=False) + _data -= self.nd_mean + _data *= 0.0078125 + if self.cutoff > 0: + _rd = random.randint(0, 1) + if _rd == 1: + #print('do cutoff aug', self.cutoff) + centerh = random.randint(0, _data.shape[0] - 1) + centerw = random.randint(0, _data.shape[1] - 1) + half = self.cutoff // 2 + starth = max(0, centerh - half) + endh = min(_data.shape[0], centerh + half) + startw = max(0, centerw - half) + endw = min(_data.shape[1], centerw + half) + #print(starth, endh, startw, endw, _data.shape) + _data[starth:endh, startw:endw, :] = 128 data = [_data] try: self.check_valid_image(data) @@ -245,7 +251,7 @@ class FaceImageIter(io.DataIter): batch_label[i][:] = label i += 1 except StopIteration: - if i0 - self.provide_data = iter_list[0].provide_data - self.provide_label = iter_list[0].provide_label - self.iter_list = iter_list - self.cur_iter = None + def __init__(self, iter_list): + assert len(iter_list) > 0 + self.provide_data = iter_list[0].provide_data + self.provide_label = iter_list[0].provide_label + self.iter_list = iter_list + self.cur_iter = None - def reset(self): - self.cur_iter.reset() - - def next(self): - self.cur_iter = random.choice(self.iter_list) - while True: - try: - ret = self.cur_iter.next() - except StopIteration: + def reset(self): self.cur_iter.reset() - continue - return ret - + def next(self): + self.cur_iter = random.choice(self.iter_list) + while True: + try: + ret = self.cur_iter.next() + except StopIteration: + self.cur_iter.reset() + continue + return ret diff --git a/recognition/ArcFace/metric.py b/recognition/ArcFace/metric.py index 0758c52..9a64c16 100644 --- a/recognition/ArcFace/metric.py +++ b/recognition/ArcFace/metric.py @@ -1,46 +1,50 @@ import numpy as np import mxnet as mx -class AccMetric(mx.metric.EvalMetric): - def __init__(self): - self.axis = 1 - super(AccMetric, self).__init__( - 'acc', axis=self.axis, - output_names=None, label_names=None) - self.losses = [] - self.count = 0 - def update(self, labels, preds): - self.count+=1 - label = labels[0] - pred_label = preds[1] - #print('ACC', label.shape, pred_label.shape) - if pred_label.shape != label.shape: - pred_label = mx.ndarray.argmax(pred_label, axis=self.axis) - pred_label = pred_label.asnumpy().astype('int32').flatten() - label = label.asnumpy() - if label.ndim==2: - label = label[:,0] - label = label.astype('int32').flatten() - assert label.shape==pred_label.shape - self.sum_metric += (pred_label.flat == label.flat).sum() - self.num_inst += len(pred_label.flat) +class AccMetric(mx.metric.EvalMetric): + def __init__(self): + self.axis = 1 + super(AccMetric, self).__init__('acc', + axis=self.axis, + output_names=None, + label_names=None) + self.losses = [] + self.count = 0 + + def update(self, labels, preds): + self.count += 1 + label = labels[0] + pred_label = preds[1] + #print('ACC', label.shape, pred_label.shape) + if pred_label.shape != label.shape: + pred_label = mx.ndarray.argmax(pred_label, axis=self.axis) + pred_label = pred_label.asnumpy().astype('int32').flatten() + label = label.asnumpy() + if label.ndim == 2: + label = label[:, 0] + label = label.astype('int32').flatten() + assert label.shape == pred_label.shape + self.sum_metric += (pred_label.flat == label.flat).sum() + self.num_inst += len(pred_label.flat) + class LossValueMetric(mx.metric.EvalMetric): - def __init__(self): - self.axis = 1 - super(LossValueMetric, self).__init__( - 'lossvalue', axis=self.axis, - output_names=None, label_names=None) - self.losses = [] + def __init__(self): + self.axis = 1 + super(LossValueMetric, self).__init__('lossvalue', + axis=self.axis, + output_names=None, + label_names=None) + self.losses = [] - def update(self, labels, preds): - #label = labels[0].asnumpy() - pred = preds[-1].asnumpy() - #print('in loss', pred.shape) - #print(pred) - loss = pred[0] - self.sum_metric += loss - self.num_inst += 1.0 - #gt_label = preds[-2].asnumpy() - #print(gt_label) + def update(self, labels, preds): + #label = labels[0].asnumpy() + pred = preds[-1].asnumpy() + #print('in loss', pred.shape) + #print(pred) + loss = pred[0] + self.sum_metric += loss + self.num_inst += 1.0 + #gt_label = preds[-2].asnumpy() + #print(gt_label) diff --git a/recognition/ArcFace/parall_module_local_v1.py b/recognition/ArcFace/parall_module_local_v1.py index 0963eb0..a3734ee 100644 --- a/recognition/ArcFace/parall_module_local_v1.py +++ b/recognition/ArcFace/parall_module_local_v1.py @@ -1,4 +1,3 @@ - ''' @author: insightface ''' @@ -20,11 +19,17 @@ from mxnet import io import mxnet.ndarray as nd from config import config + class ParallModule(BaseModule): - def __init__(self, symbol, data_names, label_names, - logger=logging, context=ctx.cpu(), work_load_list=None, - asymbol = None, - args = None): + def __init__(self, + symbol, + data_names, + label_names, + logger=logging, + context=ctx.cpu(), + work_load_list=None, + asymbol=None, + args=None): super(ParallModule, self).__init__(logger=logger) self._symbol = symbol self._asymbol = asymbol @@ -48,25 +53,32 @@ class ParallModule(BaseModule): self._ctx_cpu = mx.cpu() self._ctx_single_gpu = self._context[-1] self._fixed_param_names = None - self._curr_module = Module(self._symbol, self._data_names, self._label_names, logger=self.logger, - context=self._context, work_load_list=self._work_load_list, - fixed_param_names=self._fixed_param_names) + self._curr_module = Module(self._symbol, + self._data_names, + self._label_names, + logger=self.logger, + context=self._context, + work_load_list=self._work_load_list, + fixed_param_names=self._fixed_param_names) self._arcface_modules = [] self._ctx_class_start = [] for i in range(len(self._context)): - args._ctxid = i - _module = Module(self._asymbol(args), self._data_names, self._label_names, logger=self.logger, - context=mx.gpu(i), work_load_list=self._work_load_list, - fixed_param_names=self._fixed_param_names) - self._arcface_modules.append(_module) - _c = args.local_class_start + i*args.ctx_num_classes - self._ctx_class_start.append(_c) + args._ctxid = i + _module = Module(self._asymbol(args), + self._data_names, + self._label_names, + logger=self.logger, + context=mx.gpu(i), + work_load_list=self._work_load_list, + fixed_param_names=self._fixed_param_names) + self._arcface_modules.append(_module) + _c = args.local_class_start + i * args.ctx_num_classes + self._ctx_class_start.append(_c) self._usekv = False if self._usekv: - self._distkv = mx.kvstore.create('dist_sync') - self._kvinit = {} - + self._distkv = mx.kvstore.create('dist_sync') + self._kvinit = {} def _reset_bind(self): self.binded = False @@ -108,54 +120,72 @@ class ParallModule(BaseModule): g = _g.copy() x = _x.copy() for _module in self._arcface_modules: - _g, _x = _module.get_params() - ag = _g.copy() - ax = _x.copy() - g.update(ag) - x.update(ax) + _g, _x = _module.get_params() + ag = _g.copy() + ax = _x.copy() + g.update(ag) + x.update(ax) return g, x - def set_params(self, arg_params, aux_params, allow_missing=False, force_init=True, + def set_params(self, + arg_params, + aux_params, + allow_missing=False, + force_init=True, allow_extra=False): - g = arg_params - x = aux_params - #ag = {} - #ax = {} - rk = [] - for k in g: - v = g[k] - if k.startswith('fc7'): - p1 = k.find('_') - p2 = k.rfind('_') - _ctxid = int(k[p1+1:p2]) - self._arcface_modules[_ctxid].set_params({k:v}, {}) - rk.append(k) - for k in rk: - del g[k] - self._curr_module.set_params(g, x) - #self._arcface_module.set_params(ag, ax) + g = arg_params + x = aux_params + #ag = {} + #ax = {} + rk = [] + for k in g: + v = g[k] + if k.startswith('fc7'): + p1 = k.find('_') + p2 = k.rfind('_') + _ctxid = int(k[p1 + 1:p2]) + self._arcface_modules[_ctxid].set_params({k: v}, {}) + rk.append(k) + for k in rk: + del g[k] + self._curr_module.set_params(g, x) + #self._arcface_module.set_params(ag, ax) - - def init_params(self, initializer=Uniform(0.01), arg_params=None, aux_params=None, - allow_missing=False, force_init=False, allow_extra=False): + def init_params(self, + initializer=Uniform(0.01), + arg_params=None, + aux_params=None, + allow_missing=False, + force_init=False, + allow_extra=False): if self.params_initialized and not force_init: return assert self.binded, 'call bind before initializing the parameters' #TODO init the same weights with all work nodes - self._curr_module.init_params(initializer=initializer, arg_params=None, - aux_params=None, allow_missing=allow_missing, - force_init=force_init, allow_extra=allow_extra) + self._curr_module.init_params(initializer=initializer, + arg_params=arg_params, + aux_params=aux_params, + allow_missing=allow_missing, + force_init=force_init, + allow_extra=allow_extra) for _module in self._arcface_modules: - #_initializer = initializer - _initializer = mx.init.Normal(0.01) - _module.init_params(initializer=_initializer, arg_params=None, - aux_params=None, allow_missing=allow_missing, - force_init=force_init, allow_extra=allow_extra) + #_initializer = initializer + _initializer = mx.init.Normal(0.01) + _module.init_params(initializer=_initializer, + arg_params=None, + aux_params=None, + allow_missing=allow_missing, + force_init=force_init, + allow_extra=allow_extra) self.params_initialized = True - - def bind(self, data_shapes, label_shapes=None, for_training=True, - inputs_need_grad=False, force_rebind=False, shared_module=None): + def bind(self, + data_shapes, + label_shapes=None, + for_training=True, + inputs_need_grad=False, + force_rebind=False, + shared_module=None): print('in_bind', self.params_initialized, data_shapes, label_shapes) if self.params_initialized: arg_params, aux_params = self.get_params() @@ -173,37 +203,54 @@ class ParallModule(BaseModule): self.for_training = for_training self.inputs_need_grad = inputs_need_grad self.binded = True - self._curr_module.bind(data_shapes, label_shapes, for_training, inputs_need_grad, - force_rebind=False, shared_module=None) + self._curr_module.bind(data_shapes, + label_shapes, + for_training, + inputs_need_grad, + force_rebind=False, + shared_module=None) _data_shape = data_shapes[0][1] print('_data_shape', _data_shape, label_shapes) for _module in self._arcface_modules: - _module.bind([('data', (_data_shape[0]*self._num_workers, self._emb_size))], [('softmax_label', (_data_shape[0]*self._num_workers,))], for_training, True, - force_rebind=False, shared_module=None) + _module.bind( + [('data', + (_data_shape[0] * self._num_workers, self._emb_size))], + [('softmax_label', (_data_shape[0] * self._num_workers, ))], + for_training, + True, + force_rebind=False, + shared_module=None) if self.params_initialized: self.set_params(arg_params, aux_params) - def init_optimizer(self, kvstore='local', optimizer='sgd', - optimizer_params=(('learning_rate', 0.01),), force_init=False): + def init_optimizer(self, + kvstore='local', + optimizer='sgd', + optimizer_params=(('learning_rate', 0.01), ), + force_init=False): assert self.binded and self.params_initialized if self.optimizer_initialized and not force_init: self.logger.warning('optimizer already initialized, ignoring.') return - self._curr_module.init_optimizer(kvstore, optimizer, optimizer_params, + self._curr_module.init_optimizer(kvstore, + optimizer, + optimizer_params, force_init=force_init) for _module in self._arcface_modules: - _module.init_optimizer(kvstore, optimizer, optimizer_params, - force_init=force_init) + _module.init_optimizer(kvstore, + optimizer, + optimizer_params, + force_init=force_init) self.optimizer_initialized = True def kv_push(self, key, value): - #if value.context!=mx.cpu(): - # value = value.as_in_context(mx.cpu()) - if not key in self._kvinit: - self._distkv.init(key, nd.zeros_like(value)) - self._kvinit[key] = 1 - self._distkv.push(key, value) + #if value.context!=mx.cpu(): + # value = value.as_in_context(mx.cpu()) + if not key in self._kvinit: + self._distkv.init(key, nd.zeros_like(value)) + self._kvinit[key] = 1 + self._distkv.push(key, value) #get fc1 and partial fc7 def forward(self, data_batch, is_train=None): @@ -211,43 +258,41 @@ class ParallModule(BaseModule): #print('{fc7_weight[0][0]}', self._iter, g['fc7_0_weight'].asnumpy()[0][0]) #print('{pre_fc1_weight[0][0]}', self._iter, g['pre_fc1_weight'].asnumpy()[0][0]) - assert self.binded and self.params_initialized self._curr_module.forward(data_batch, is_train=is_train) if is_train: - self._iter+=1 - fc1, label = self._curr_module.get_outputs(merge_multi_context=True) - global_fc1 = fc1 - self.global_label = label.as_in_context(self._ctx_cpu) + self._iter += 1 + fc1, label = self._curr_module.get_outputs( + merge_multi_context=True) + global_fc1 = fc1 + self.global_label = label.as_in_context(self._ctx_cpu) - - for i, _module in enumerate(self._arcface_modules): - _label = self.global_label - self._ctx_class_start[i] - db_global_fc1 = io.DataBatch([global_fc1], [_label]) - _module.forward(db_global_fc1) #fc7 with margin + for i, _module in enumerate(self._arcface_modules): + _label = self.global_label - self._ctx_class_start[i] + db_global_fc1 = io.DataBatch([global_fc1], [_label]) + _module.forward(db_global_fc1) #fc7 with margin #print('forward end') - def get_ndarray(self, context, name, shape): - key = "%s_%s"%(name, context) - #print(key) - if not key in self._nd_cache: - v = nd.zeros( shape=shape, ctx = context) - self._nd_cache[key] = v - else: - v = self._nd_cache[key] - return v + key = "%s_%s" % (name, context) + #print(key) + if not key in self._nd_cache: + v = nd.zeros(shape=shape, ctx=context) + self._nd_cache[key] = v + else: + v = self._nd_cache[key] + return v def get_ndarray2(self, context, name, arr): - key = "%s_%s"%(name, context) - #print(key) - if not key in self._nd_cache: - v = nd.zeros( shape=arr.shape, ctx = context) - self._nd_cache[key] = v - else: - v = self._nd_cache[key] - arr.copyto(v) - return v + key = "%s_%s" % (name, context) + #print(key) + if not key in self._nd_cache: + v = nd.zeros(shape=arr.shape, ctx=context) + self._nd_cache[key] = v + else: + v = self._nd_cache[key] + arr.copyto(v) + return v def backward(self, out_grads=None): #print('in backward') @@ -255,85 +300,102 @@ class ParallModule(BaseModule): #tmp_ctx = self._ctx_cpu tmp_ctx = self._ctx_single_gpu fc7_outs = [] - ctx_fc7_max = self.get_ndarray(tmp_ctx, 'ctx_fc7_max', (self._batch_size, len(self._context))) + ctx_fc7_max = self.get_ndarray(tmp_ctx, 'ctx_fc7_max', + (self._batch_size, len(self._context))) #local_fc7_max = nd.zeros( (self.global_label.shape[0],1), ctx=mx.cpu()) for i, _module in enumerate(self._arcface_modules): - _fc7 = _module.get_outputs(merge_multi_context=True)[0] - fc7_outs.append(_fc7) - _fc7_max = nd.max(_fc7, axis=1).as_in_context(tmp_ctx) - ctx_fc7_max[:,i] = _fc7_max + _fc7 = _module.get_outputs(merge_multi_context=True)[0] + fc7_outs.append(_fc7) + _fc7_max = nd.max(_fc7, axis=1).as_in_context(tmp_ctx) + ctx_fc7_max[:, i] = _fc7_max - local_fc7_max = self.get_ndarray(tmp_ctx, 'local_fc7_max', (self._batch_size, 1)) + local_fc7_max = self.get_ndarray(tmp_ctx, 'local_fc7_max', + (self._batch_size, 1)) nd.max(ctx_fc7_max, axis=1, keepdims=True, out=local_fc7_max) global_fc7_max = local_fc7_max #local_fc7_sum = None - local_fc7_sum = self.get_ndarray(tmp_ctx, 'local_fc7_sum', (self._batch_size,1)) - local_fc7_sum[:,:] = 0.0 + local_fc7_sum = self.get_ndarray(tmp_ctx, 'local_fc7_sum', + (self._batch_size, 1)) + local_fc7_sum[:, :] = 0.0 for i, _module in enumerate(self._arcface_modules): - _max = self.get_ndarray2(fc7_outs[i].context, 'fc7_max', global_fc7_max) - fc7_outs[i] = nd.broadcast_sub(fc7_outs[i], _max) - fc7_outs[i] = nd.exp(fc7_outs[i]) - _sum = nd.sum(fc7_outs[i], axis=1, keepdims=True).as_in_context(tmp_ctx) - local_fc7_sum += _sum + _max = self.get_ndarray2(fc7_outs[i].context, 'fc7_max', + global_fc7_max) + fc7_outs[i] = nd.broadcast_sub(fc7_outs[i], _max) + fc7_outs[i] = nd.exp(fc7_outs[i]) + _sum = nd.sum(fc7_outs[i], axis=1, + keepdims=True).as_in_context(tmp_ctx) + local_fc7_sum += _sum global_fc7_sum = local_fc7_sum - if self._iter%self._verbose==0: - #_ctx = self._context[-1] - _ctx = self._ctx_cpu - _probs = [] - for i, _module in enumerate(self._arcface_modules): - _prob = self.get_ndarray2(_ctx, '_fc7_prob_%d'%i, fc7_outs[i]) - _probs.append(_prob) - fc7_prob = self.get_ndarray(_ctx, 'test_fc7_prob', (self._batch_size, self._ctx_num_classes*len(self._context))) - nd.concat(*_probs, dim=1, out=fc7_prob) - fc7_pred = nd.argmax(fc7_prob, axis=1) - local_label = self.global_label - self._local_class_start - #local_label = self.get_ndarray2(_ctx, 'test_label', local_label) - _pred = nd.equal(fc7_pred, local_label) - print('{fc7_acc}', self._iter, nd.mean(_pred).asnumpy()[0]) - + if self._iter % self._verbose == 0: + #_ctx = self._context[-1] + _ctx = self._ctx_cpu + _probs = [] + for i, _module in enumerate(self._arcface_modules): + _prob = self.get_ndarray2(_ctx, '_fc7_prob_%d' % i, + fc7_outs[i]) + _probs.append(_prob) + fc7_prob = self.get_ndarray( + _ctx, 'test_fc7_prob', + (self._batch_size, self._ctx_num_classes * len(self._context))) + nd.concat(*_probs, dim=1, out=fc7_prob) + fc7_pred = nd.argmax(fc7_prob, axis=1) + local_label = self.global_label - self._local_class_start + #local_label = self.get_ndarray2(_ctx, 'test_label', local_label) + _pred = nd.equal(fc7_pred, local_label) + print('{fc7_acc}', self._iter, nd.mean(_pred).asnumpy()[0]) #local_fc1_grad = [] #fc1_grad_ctx = self._ctx_cpu fc1_grad_ctx = self._ctx_single_gpu - local_fc1_grad = self.get_ndarray(fc1_grad_ctx, 'local_fc1_grad', (self._batch_size,self._emb_size)) - local_fc1_grad[:,:] = 0.0 + local_fc1_grad = self.get_ndarray(fc1_grad_ctx, 'local_fc1_grad', + (self._batch_size, self._emb_size)) + local_fc1_grad[:, :] = 0.0 for i, _module in enumerate(self._arcface_modules): - _sum = self.get_ndarray2(fc7_outs[i].context, 'fc7_sum', global_fc7_sum) - fc7_outs[i] = nd.broadcast_div(fc7_outs[i], _sum) - a = i*self._ctx_num_classes - b = (i+1)*self._ctx_num_classes - _label = self.global_label - self._ctx_class_start[i] - _label = self.get_ndarray2(fc7_outs[i].context, 'label', _label) - onehot_label = self.get_ndarray(fc7_outs[i].context, 'label_onehot', (self._batch_size, self._ctx_num_classes)) - nd.one_hot(_label, depth=self._ctx_num_classes, on_value = 1.0, off_value = 0.0, out=onehot_label) - fc7_outs[i] -= onehot_label - _module.backward(out_grads = [fc7_outs[i]]) - #ctx_fc1_grad = _module.get_input_grads()[0].as_in_context(mx.cpu()) - ctx_fc1_grad = self.get_ndarray2(fc1_grad_ctx, 'ctx_fc1_grad_%d'%i, _module.get_input_grads()[0]) - local_fc1_grad += ctx_fc1_grad + _sum = self.get_ndarray2(fc7_outs[i].context, 'fc7_sum', + global_fc7_sum) + fc7_outs[i] = nd.broadcast_div(fc7_outs[i], _sum) + a = i * self._ctx_num_classes + b = (i + 1) * self._ctx_num_classes + _label = self.global_label - self._ctx_class_start[i] + _label = self.get_ndarray2(fc7_outs[i].context, 'label', _label) + onehot_label = self.get_ndarray( + fc7_outs[i].context, 'label_onehot', + (self._batch_size, self._ctx_num_classes)) + nd.one_hot(_label, + depth=self._ctx_num_classes, + on_value=1.0, + off_value=0.0, + out=onehot_label) + fc7_outs[i] -= onehot_label + _module.backward(out_grads=[fc7_outs[i]]) + #ctx_fc1_grad = _module.get_input_grads()[0].as_in_context(mx.cpu()) + ctx_fc1_grad = self.get_ndarray2(fc1_grad_ctx, + 'ctx_fc1_grad_%d' % i, + _module.get_input_grads()[0]) + local_fc1_grad += ctx_fc1_grad global_fc1_grad = local_fc1_grad - self._curr_module.backward(out_grads = [global_fc1_grad]) - + self._curr_module.backward(out_grads=[global_fc1_grad]) def update(self): assert self.binded and self.params_initialized and self.optimizer_initialized self._curr_module.update() for i, _module in enumerate(self._arcface_modules): - _module.update() + _module.update() mx.nd.waitall() - def get_outputs(self, merge_multi_context=True): assert self.binded and self.params_initialized - return self._curr_module.get_outputs(merge_multi_context=merge_multi_context) + return self._curr_module.get_outputs( + merge_multi_context=merge_multi_context) #return self._arcface_module.get_outputs(merge_multi_context=merge_multi_context) def get_input_grads(self, merge_multi_context=True): assert self.binded and self.params_initialized and self.inputs_need_grad - return self._curr_module.get_input_grads(merge_multi_context=merge_multi_context) + return self._curr_module.get_input_grads( + merge_multi_context=merge_multi_context) def update_metric(self, eval_metric, labels): assert self.binded and self.params_initialized @@ -349,17 +411,31 @@ class ParallModule(BaseModule): def forward_backward(self, data_batch): """A convenient function that calls both ``forward`` and ``backward``.""" - self.forward(data_batch, is_train=True) # get fc1 and partial fc7 + self.forward(data_batch, is_train=True) # get fc1 and partial fc7 self.backward() - def fit(self, train_data, eval_data=None, eval_metric='acc', - epoch_end_callback=None, batch_end_callback=None, kvstore='local', - optimizer='sgd', optimizer_params=(('learning_rate', 0.01),), + def fit(self, + train_data, + eval_data=None, + eval_metric='acc', + epoch_end_callback=None, + batch_end_callback=None, + kvstore='local', + optimizer='sgd', + optimizer_params=(('learning_rate', 0.01), ), eval_end_callback=None, - eval_batch_end_callback=None, initializer=Uniform(0.01), - arg_params=None, aux_params=None, allow_missing=False, - force_rebind=False, force_init=False, begin_epoch=0, num_epoch=None, - validation_metric=None, monitor=None, sparse_row_id_fn=None): + eval_batch_end_callback=None, + initializer=Uniform(0.01), + arg_params=None, + aux_params=None, + allow_missing=False, + force_rebind=False, + force_init=False, + begin_epoch=0, + num_epoch=None, + validation_metric=None, + monitor=None, + sparse_row_id_fn=None): """Trains the module parameters. Checkout `Module Tutorial `_ to see @@ -441,13 +517,19 @@ class ParallModule(BaseModule): assert num_epoch is not None, 'please specify number of epochs' assert arg_params is None and aux_params is None - self.bind(data_shapes=train_data.provide_data, label_shapes=train_data.provide_label, - for_training=True, force_rebind=force_rebind) + self.bind(data_shapes=train_data.provide_data, + label_shapes=train_data.provide_label, + for_training=True, + force_rebind=force_rebind) if monitor is not None: self.install_monitor(monitor) - self.init_params(initializer=initializer, arg_params=arg_params, aux_params=aux_params, - allow_missing=allow_missing, force_init=force_init) - self.init_optimizer(kvstore=kvstore, optimizer=optimizer, + self.init_params(initializer=initializer, + arg_params=arg_params, + aux_params=aux_params, + allow_missing=allow_missing, + force_init=force_init) + self.init_optimizer(kvstore=kvstore, + optimizer=optimizer, optimizer_params=optimizer_params) if validation_metric is None: @@ -495,7 +577,8 @@ class ParallModule(BaseModule): try: # pre fetch next batch next_data_batch = next(data_iter) - self.prepare(next_data_batch, sparse_row_id_fn=sparse_row_id_fn) + self.prepare(next_data_batch, + sparse_row_id_fn=sparse_row_id_fn) except StopIteration: end_of_batch = True @@ -506,7 +589,8 @@ class ParallModule(BaseModule): # eval_name_vals = epoch_eval_metric.get_name_value() if batch_end_callback is not None: - batch_end_params = BatchEndParam(epoch=epoch, nbatch=nbatch, + batch_end_params = BatchEndParam(epoch=epoch, + nbatch=nbatch, eval_metric=None, locals=locals()) batch_end_callback(batch_end_params) @@ -518,7 +602,7 @@ class ParallModule(BaseModule): #for name, val in eval_name_vals: # self.logger.info('Epoch[%d] Train-%s=%f', epoch, name, val) toc = time.time() - self.logger.info('Epoch[%d] Time cost=%.3f', epoch, (toc-tic)) + self.logger.info('Epoch[%d] Time cost=%.3f', epoch, (toc - tic)) # sync aux params across devices arg_params, aux_params = self.get_params() @@ -526,4 +610,3 @@ class ParallModule(BaseModule): # end of 1 epoch, reset the data-iter for another epoch train_data.reset() - diff --git a/recognition/ArcFace/sample_config.py b/recognition/ArcFace/sample_config.py index d91887c..f7eff62 100644 --- a/recognition/ArcFace/sample_config.py +++ b/recognition/ArcFace/sample_config.py @@ -12,7 +12,7 @@ config.net_se = 0 config.net_act = 'prelu' config.net_unit = 3 config.net_input = 1 -config.net_blocks = [1,4,6,2] +config.net_blocks = [1, 4, 6, 2] config.net_output = 'E' config.net_multiplier = 1.0 config.val_targets = ['lfw', 'cfp_fp', 'agedb_30'] @@ -26,8 +26,7 @@ config.data_cutoff = False config.data_color = 0 config.data_images_filter = 0 config.count_flops = True -config.memonger = False #not work now - +config.memonger = False #not work now # network settings network = edict() @@ -71,7 +70,7 @@ network.y2 = edict() network.y2.net_name = 'fmobilefacenet' network.y2.emb_size = 256 network.y2.net_output = 'GDC' -network.y2.net_blocks = [2,8,16,4] +network.y2.net_blocks = [2, 8, 16, 4] network.m1 = edict() network.m1.net_name = 'fmobilenet' @@ -107,7 +106,7 @@ network.vargfacenet = edict() network.vargfacenet.net_name = 'vargfacenet' network.vargfacenet.net_multiplier = 1.25 network.vargfacenet.emb_size = 512 -network.vargfacenet.net_output='J' +network.vargfacenet.net_output = 'J' # dataset settings dataset = edict() @@ -116,14 +115,14 @@ dataset.emore = edict() dataset.emore.dataset = 'emore' dataset.emore.dataset_path = '../datasets/faces_emore' dataset.emore.num_classes = 85742 -dataset.emore.image_shape = (112,112,3) +dataset.emore.image_shape = (112, 112, 3) dataset.emore.val_targets = ['lfw', 'cfp_fp', 'agedb_30'] dataset.retina = edict() dataset.retina.dataset = 'retina' dataset.retina.dataset_path = '../datasets/ms1m-retinaface-t1' dataset.retina.num_classes = 93431 -dataset.retina.image_shape = (112,112,3) +dataset.retina.image_shape = (112, 112, 3) dataset.retina.val_targets = ['lfw', 'cfp_fp', 'agedb_30'] loss = edict() @@ -202,21 +201,20 @@ default.models_root = './models' def generate_config(_network, _dataset, _loss): for k, v in loss[_loss].items(): - config[k] = v - if k in default: - default[k] = v + config[k] = v + if k in default: + default[k] = v for k, v in network[_network].items(): - config[k] = v - if k in default: - default[k] = v + config[k] = v + if k in default: + default[k] = v for k, v in dataset[_dataset].items(): - config[k] = v - if k in default: - default[k] = v + config[k] = v + if k in default: + default[k] = v config.loss = _loss config.network = _network config.dataset = _dataset config.num_workers = 1 if 'DMLC_NUM_WORKER' in os.environ: - config.num_workers = int(os.environ['DMLC_NUM_WORKER']) - + config.num_workers = int(os.environ['DMLC_NUM_WORKER']) diff --git a/recognition/ArcFace/train.py b/recognition/ArcFace/train.py index 9216f9a..393426f 100644 --- a/recognition/ArcFace/train.py +++ b/recognition/ArcFace/train.py @@ -27,142 +27,232 @@ import fmnasnet import fdensenet import vargfacenet - logger = logging.getLogger() logger.setLevel(logging.INFO) - args = None - def parse_args(): - parser = argparse.ArgumentParser(description='Train face network') - # general - parser.add_argument('--dataset', default=default.dataset, help='dataset config') - parser.add_argument('--network', default=default.network, help='network config') - parser.add_argument('--loss', default=default.loss, help='loss config') - args, rest = parser.parse_known_args() - generate_config(args.network, args.dataset, args.loss) - parser.add_argument('--models-root', default=default.models_root, help='root directory to save model.') - parser.add_argument('--pretrained', default=default.pretrained, help='pretrained model to load') - parser.add_argument('--pretrained-epoch', type=int, default=default.pretrained_epoch, help='pretrained epoch to load') - parser.add_argument('--ckpt', type=int, default=default.ckpt, help='checkpoint saving option. 0: discard saving. 1: save when necessary. 2: always save') - parser.add_argument('--verbose', type=int, default=default.verbose, help='do verification testing and model saving every verbose batches') - parser.add_argument('--lr', type=float, default=default.lr, help='start learning rate') - parser.add_argument('--lr-steps', type=str, default=default.lr_steps, help='steps of lr changing') - parser.add_argument('--wd', type=float, default=default.wd, help='weight decay') - parser.add_argument('--mom', type=float, default=default.mom, help='momentum') - parser.add_argument('--frequent', type=int, default=default.frequent, help='') - parser.add_argument('--per-batch-size', type=int, default=default.per_batch_size, help='batch size in each context') - parser.add_argument('--kvstore', type=str, default=default.kvstore, help='kvstore setting') - args = parser.parse_args() - return args + parser = argparse.ArgumentParser(description='Train face network') + # general + parser.add_argument('--dataset', + default=default.dataset, + help='dataset config') + parser.add_argument('--network', + default=default.network, + help='network config') + parser.add_argument('--loss', default=default.loss, help='loss config') + args, rest = parser.parse_known_args() + generate_config(args.network, args.dataset, args.loss) + parser.add_argument('--models-root', + default=default.models_root, + help='root directory to save model.') + parser.add_argument('--pretrained', + default=default.pretrained, + help='pretrained model to load') + parser.add_argument('--pretrained-epoch', + type=int, + default=default.pretrained_epoch, + help='pretrained epoch to load') + parser.add_argument( + '--ckpt', + type=int, + default=default.ckpt, + help= + 'checkpoint saving option. 0: discard saving. 1: save when necessary. 2: always save' + ) + parser.add_argument( + '--verbose', + type=int, + default=default.verbose, + help='do verification testing and model saving every verbose batches') + parser.add_argument('--lr', + type=float, + default=default.lr, + help='start learning rate') + parser.add_argument('--lr-steps', + type=str, + default=default.lr_steps, + help='steps of lr changing') + parser.add_argument('--wd', + type=float, + default=default.wd, + help='weight decay') + parser.add_argument('--mom', + type=float, + default=default.mom, + help='momentum') + parser.add_argument('--frequent', + type=int, + default=default.frequent, + help='') + parser.add_argument('--per-batch-size', + type=int, + default=default.per_batch_size, + help='batch size in each context') + parser.add_argument('--kvstore', + type=str, + default=default.kvstore, + help='kvstore setting') + args = parser.parse_args() + return args def get_symbol(args): - embedding = eval(config.net_name).get_symbol() - all_label = mx.symbol.Variable('softmax_label') - gt_label = all_label - is_softmax = True - if config.loss_name=='softmax': #softmax - _weight = mx.symbol.Variable("fc7_weight", shape=(config.num_classes, config.emb_size), - lr_mult=config.fc7_lr_mult, wd_mult=config.fc7_wd_mult, init=mx.init.Normal(0.01)) - if config.fc7_no_bias: - fc7 = mx.sym.FullyConnected(data=embedding, weight = _weight, no_bias = True, num_hidden=config.num_classes, name='fc7') + embedding = eval(config.net_name).get_symbol() + all_label = mx.symbol.Variable('softmax_label') + gt_label = all_label + is_softmax = True + if config.loss_name == 'softmax': #softmax + _weight = mx.symbol.Variable("fc7_weight", + shape=(config.num_classes, + config.emb_size), + lr_mult=config.fc7_lr_mult, + wd_mult=config.fc7_wd_mult, + init=mx.init.Normal(0.01)) + if config.fc7_no_bias: + fc7 = mx.sym.FullyConnected(data=embedding, + weight=_weight, + no_bias=True, + num_hidden=config.num_classes, + name='fc7') + else: + _bias = mx.symbol.Variable('fc7_bias', lr_mult=2.0, wd_mult=0.0) + fc7 = mx.sym.FullyConnected(data=embedding, + weight=_weight, + bias=_bias, + num_hidden=config.num_classes, + name='fc7') + elif config.loss_name == 'margin_softmax': + _weight = mx.symbol.Variable("fc7_weight", + shape=(config.num_classes, + config.emb_size), + lr_mult=config.fc7_lr_mult, + wd_mult=config.fc7_wd_mult, + init=mx.init.Normal(0.01)) + s = config.loss_s + _weight = mx.symbol.L2Normalization(_weight, mode='instance') + nembedding = mx.symbol.L2Normalization( + embedding, mode='instance', name='fc1n') * s + fc7 = mx.sym.FullyConnected(data=nembedding, + weight=_weight, + no_bias=True, + num_hidden=config.num_classes, + name='fc7') + if config.loss_m1 != 1.0 or config.loss_m2 != 0.0 or config.loss_m3 != 0.0: + if config.loss_m1 == 1.0 and config.loss_m2 == 0.0: + s_m = s * config.loss_m3 + gt_one_hot = mx.sym.one_hot(gt_label, + depth=config.num_classes, + on_value=s_m, + off_value=0.0) + fc7 = fc7 - gt_one_hot + else: + zy = mx.sym.pick(fc7, gt_label, axis=1) + cos_t = zy / s + t = mx.sym.arccos(cos_t) + if config.loss_m1 != 1.0: + t = t * config.loss_m1 + if config.loss_m2 > 0.0: + t = t + config.loss_m2 + body = mx.sym.cos(t) + if config.loss_m3 > 0.0: + body = body - config.loss_m3 + new_zy = body * s + diff = new_zy - zy + diff = mx.sym.expand_dims(diff, 1) + gt_one_hot = mx.sym.one_hot(gt_label, + depth=config.num_classes, + on_value=1.0, + off_value=0.0) + body = mx.sym.broadcast_mul(gt_one_hot, diff) + fc7 = fc7 + body + elif config.loss_name.find('triplet') >= 0: + is_softmax = False + nembedding = mx.symbol.L2Normalization(embedding, + mode='instance', + name='fc1n') + anchor = mx.symbol.slice_axis(nembedding, + axis=0, + begin=0, + end=args.per_batch_size // 3) + positive = mx.symbol.slice_axis(nembedding, + axis=0, + begin=args.per_batch_size // 3, + end=2 * args.per_batch_size // 3) + negative = mx.symbol.slice_axis(nembedding, + axis=0, + begin=2 * args.per_batch_size // 3, + end=args.per_batch_size) + if config.loss_name == 'triplet': + ap = anchor - positive + an = anchor - negative + ap = ap * ap + an = an * an + ap = mx.symbol.sum(ap, axis=1, keepdims=1) #(T,1) + an = mx.symbol.sum(an, axis=1, keepdims=1) #(T,1) + triplet_loss = mx.symbol.Activation(data=(ap - an + + config.triplet_alpha), + act_type='relu') + triplet_loss = mx.symbol.mean(triplet_loss) + else: + ap = anchor * positive + an = anchor * negative + ap = mx.symbol.sum(ap, axis=1, keepdims=1) #(T,1) + an = mx.symbol.sum(an, axis=1, keepdims=1) #(T,1) + ap = mx.sym.arccos(ap) + an = mx.sym.arccos(an) + triplet_loss = mx.symbol.Activation(data=(ap - an + + config.triplet_alpha), + act_type='relu') + triplet_loss = mx.symbol.mean(triplet_loss) + triplet_loss = mx.symbol.MakeLoss(triplet_loss) + out_list = [mx.symbol.BlockGrad(embedding)] + if is_softmax: + softmax = mx.symbol.SoftmaxOutput(data=fc7, + label=gt_label, + name='softmax', + normalization='valid') + out_list.append(softmax) + if config.ce_loss: + #ce_loss = mx.symbol.softmax_cross_entropy(data=fc7, label = gt_label, name='ce_loss')/args.per_batch_size + body = mx.symbol.SoftmaxActivation(data=fc7) + body = mx.symbol.log(body) + _label = mx.sym.one_hot(gt_label, + depth=config.num_classes, + on_value=-1.0, + off_value=0.0) + body = body * _label + ce_loss = mx.symbol.sum(body) / args.per_batch_size + out_list.append(mx.symbol.BlockGrad(ce_loss)) else: - _bias = mx.symbol.Variable('fc7_bias', lr_mult=2.0, wd_mult=0.0) - fc7 = mx.sym.FullyConnected(data=embedding, weight = _weight, bias = _bias, num_hidden=config.num_classes, name='fc7') - elif config.loss_name=='margin_softmax': - _weight = mx.symbol.Variable("fc7_weight", shape=(config.num_classes, config.emb_size), - lr_mult=config.fc7_lr_mult, wd_mult=config.fc7_wd_mult, init=mx.init.Normal(0.01)) - s = config.loss_s - _weight = mx.symbol.L2Normalization(_weight, mode='instance') - nembedding = mx.symbol.L2Normalization(embedding, mode='instance', name='fc1n')*s - fc7 = mx.sym.FullyConnected(data=nembedding, weight = _weight, no_bias = True, num_hidden=config.num_classes, name='fc7') - if config.loss_m1!=1.0 or config.loss_m2!=0.0 or config.loss_m3!=0.0: - if config.loss_m1==1.0 and config.loss_m2==0.0: - s_m = s*config.loss_m3 - gt_one_hot = mx.sym.one_hot(gt_label, depth = config.num_classes, on_value = s_m, off_value = 0.0) - fc7 = fc7-gt_one_hot - else: - zy = mx.sym.pick(fc7, gt_label, axis=1) - cos_t = zy/s - t = mx.sym.arccos(cos_t) - if config.loss_m1!=1.0: - t = t*config.loss_m1 - if config.loss_m2>0.0: - t = t+config.loss_m2 - body = mx.sym.cos(t) - if config.loss_m3>0.0: - body = body - config.loss_m3 - new_zy = body*s - diff = new_zy - zy - diff = mx.sym.expand_dims(diff, 1) - gt_one_hot = mx.sym.one_hot(gt_label, depth = config.num_classes, on_value = 1.0, off_value = 0.0) - body = mx.sym.broadcast_mul(gt_one_hot, diff) - fc7 = fc7+body - elif config.loss_name.find('triplet')>=0: - is_softmax = False - nembedding = mx.symbol.L2Normalization(embedding, mode='instance', name='fc1n') - anchor = mx.symbol.slice_axis(nembedding, axis=0, begin=0, end=args.per_batch_size//3) - positive = mx.symbol.slice_axis(nembedding, axis=0, begin=args.per_batch_size//3, end=2*args.per_batch_size//3) - negative = mx.symbol.slice_axis(nembedding, axis=0, begin=2*args.per_batch_size//3, end=args.per_batch_size) - if config.loss_name=='triplet': - ap = anchor - positive - an = anchor - negative - ap = ap*ap - an = an*an - ap = mx.symbol.sum(ap, axis=1, keepdims=1) #(T,1) - an = mx.symbol.sum(an, axis=1, keepdims=1) #(T,1) - triplet_loss = mx.symbol.Activation(data = (ap-an+config.triplet_alpha), act_type='relu') - triplet_loss = mx.symbol.mean(triplet_loss) - else: - ap = anchor*positive - an = anchor*negative - ap = mx.symbol.sum(ap, axis=1, keepdims=1) #(T,1) - an = mx.symbol.sum(an, axis=1, keepdims=1) #(T,1) - ap = mx.sym.arccos(ap) - an = mx.sym.arccos(an) - triplet_loss = mx.symbol.Activation(data = (ap-an+config.triplet_alpha), act_type='relu') - triplet_loss = mx.symbol.mean(triplet_loss) - triplet_loss = mx.symbol.MakeLoss(triplet_loss) - out_list = [mx.symbol.BlockGrad(embedding)] - if is_softmax: - softmax = mx.symbol.SoftmaxOutput(data=fc7, label = gt_label, name='softmax', normalization='valid') - out_list.append(softmax) - if config.ce_loss: - #ce_loss = mx.symbol.softmax_cross_entropy(data=fc7, label = gt_label, name='ce_loss')/args.per_batch_size - body = mx.symbol.SoftmaxActivation(data=fc7) - body = mx.symbol.log(body) - _label = mx.sym.one_hot(gt_label, depth = config.num_classes, on_value = -1.0, off_value = 0.0) - body = body*_label - ce_loss = mx.symbol.sum(body)/args.per_batch_size - out_list.append(mx.symbol.BlockGrad(ce_loss)) - else: - out_list.append(mx.sym.BlockGrad(gt_label)) - out_list.append(triplet_loss) - out = mx.symbol.Group(out_list) - return out + out_list.append(mx.sym.BlockGrad(gt_label)) + out_list.append(triplet_loss) + out = mx.symbol.Group(out_list) + return out + def train_net(args): ctx = [] cvd = os.environ['CUDA_VISIBLE_DEVICES'].strip() - if len(cvd)>0: - for i in range(len(cvd.split(','))): - ctx.append(mx.gpu(i)) - if len(ctx)==0: - ctx = [mx.cpu()] - print('use cpu') + if len(cvd) > 0: + for i in range(len(cvd.split(','))): + ctx.append(mx.gpu(i)) + if len(ctx) == 0: + ctx = [mx.cpu()] + print('use cpu') else: - print('gpu num:', len(ctx)) - prefix = os.path.join(args.models_root, '%s-%s-%s'%(args.network, args.loss, args.dataset), 'model') + print('gpu num:', len(ctx)) + prefix = os.path.join(args.models_root, + '%s-%s-%s' % (args.network, args.loss, args.dataset), + 'model') prefix_dir = os.path.dirname(prefix) print('prefix', prefix) if not os.path.exists(prefix_dir): - os.makedirs(prefix_dir) + os.makedirs(prefix_dir) args.ctx_num = len(ctx) - args.batch_size = args.per_batch_size*args.ctx_num + args.batch_size = args.per_batch_size * args.ctx_num args.rescale_threshold = 0 args.image_channel = config.image_shape[2] config.batch_size = args.batch_size @@ -172,113 +262,124 @@ def train_net(args): path_imgrec = None path_imglist = None image_size = config.image_shape[0:2] - assert len(image_size)==2 - assert image_size[0]==image_size[1] + assert len(image_size) == 2 + assert image_size[0] == image_size[1] print('image_size', image_size) print('num_classes', config.num_classes) path_imgrec = os.path.join(data_dir, "train.rec") print('Called with argument:', args, config) - data_shape = (args.image_channel,image_size[0],image_size[1]) + data_shape = (args.image_channel, image_size[0], image_size[1]) mean = None begin_epoch = 0 - if len(args.pretrained)==0: - arg_params = None - aux_params = None - sym = get_symbol(args) - if config.net_name=='spherenet': - data_shape_dict = {'data' : (args.per_batch_size,)+data_shape} - spherenet.init_weights(sym, data_shape_dict, args.num_layers) + if len(args.pretrained) == 0: + arg_params = None + aux_params = None + sym = get_symbol(args) + if config.net_name == 'spherenet': + data_shape_dict = {'data': (args.per_batch_size, ) + data_shape} + spherenet.init_weights(sym, data_shape_dict, args.num_layers) else: - print('loading', args.pretrained, args.pretrained_epoch) - _, arg_params, aux_params = mx.model.load_checkpoint(args.pretrained, args.pretrained_epoch) - sym = get_symbol(args) + print('loading', args.pretrained, args.pretrained_epoch) + _, arg_params, aux_params = mx.model.load_checkpoint( + args.pretrained, args.pretrained_epoch) + sym = get_symbol(args) if config.count_flops: - all_layers = sym.get_internals() - _sym = all_layers['fc1_output'] - FLOPs = flops_counter.count_flops(_sym, data=(1,3,image_size[0],image_size[1])) - _str = flops_counter.flops_str(FLOPs) - print('Network FLOPs: %s'%_str) + all_layers = sym.get_internals() + _sym = all_layers['fc1_output'] + FLOPs = flops_counter.count_flops(_sym, + data=(1, 3, image_size[0], + image_size[1])) + _str = flops_counter.flops_str(FLOPs) + print('Network FLOPs: %s' % _str) #label_name = 'softmax_label' #label_shape = (args.batch_size,) model = mx.mod.Module( - context = ctx, - symbol = sym, + context=ctx, + symbol=sym, ) val_dataiter = None - if config.loss_name.find('triplet')>=0: - from triplet_image_iter import FaceImageIter - triplet_params = [config.triplet_bag_size, config.triplet_alpha, config.triplet_max_ap] - train_dataiter = FaceImageIter( - batch_size = args.batch_size, - data_shape = data_shape, - path_imgrec = path_imgrec, - shuffle = True, - rand_mirror = config.data_rand_mirror, - mean = mean, - cutoff = config.data_cutoff, - ctx_num = args.ctx_num, - images_per_identity = config.images_per_identity, - triplet_params = triplet_params, - mx_model = model, - ) - _metric = LossValueMetric() - eval_metrics = [mx.metric.create(_metric)] + if config.loss_name.find('triplet') >= 0: + from triplet_image_iter import FaceImageIter + triplet_params = [ + config.triplet_bag_size, config.triplet_alpha, + config.triplet_max_ap + ] + train_dataiter = FaceImageIter( + batch_size=args.batch_size, + data_shape=data_shape, + path_imgrec=path_imgrec, + shuffle=True, + rand_mirror=config.data_rand_mirror, + mean=mean, + cutoff=config.data_cutoff, + ctx_num=args.ctx_num, + images_per_identity=config.images_per_identity, + triplet_params=triplet_params, + mx_model=model, + ) + _metric = LossValueMetric() + eval_metrics = [mx.metric.create(_metric)] else: - from image_iter import FaceImageIter - train_dataiter = FaceImageIter( - batch_size = args.batch_size, - data_shape = data_shape, - path_imgrec = path_imgrec, - shuffle = True, - rand_mirror = config.data_rand_mirror, - mean = mean, - cutoff = config.data_cutoff, - color_jittering = config.data_color, - images_filter = config.data_images_filter, - ) - metric1 = AccMetric() - eval_metrics = [mx.metric.create(metric1)] - if config.ce_loss: - metric2 = LossValueMetric() - eval_metrics.append( mx.metric.create(metric2) ) + from image_iter import FaceImageIter + train_dataiter = FaceImageIter( + batch_size=args.batch_size, + data_shape=data_shape, + path_imgrec=path_imgrec, + shuffle=True, + rand_mirror=config.data_rand_mirror, + mean=mean, + cutoff=config.data_cutoff, + color_jittering=config.data_color, + images_filter=config.data_images_filter, + ) + metric1 = AccMetric() + eval_metrics = [mx.metric.create(metric1)] + if config.ce_loss: + metric2 = LossValueMetric() + eval_metrics.append(mx.metric.create(metric2)) - if config.net_name=='fresnet' or config.net_name=='fmobilefacenet': - initializer = mx.init.Xavier(rnd_type='gaussian', factor_type="out", magnitude=2) #resnet style + if config.net_name == 'fresnet' or config.net_name == 'fmobilefacenet': + initializer = mx.init.Xavier(rnd_type='gaussian', + factor_type="out", + magnitude=2) #resnet style else: - initializer = mx.init.Xavier(rnd_type='uniform', factor_type="in", magnitude=2) + initializer = mx.init.Xavier(rnd_type='uniform', + factor_type="in", + magnitude=2) #initializer = mx.init.Xavier(rnd_type='gaussian', factor_type="out", magnitude=2) #resnet style - _rescale = 1.0/args.ctx_num - opt = optimizer.SGD(learning_rate=args.lr, momentum=args.mom, wd=args.wd, rescale_grad=_rescale) + _rescale = 1.0 / args.ctx_num + opt = optimizer.SGD(learning_rate=args.lr, + momentum=args.mom, + wd=args.wd, + rescale_grad=_rescale) _cb = mx.callback.Speedometer(args.batch_size, args.frequent) ver_list = [] ver_name_list = [] for name in config.val_targets: - path = os.path.join(data_dir,name+".bin") - if os.path.exists(path): - data_set = verification.load_bin(path, image_size) - ver_list.append(data_set) - ver_name_list.append(name) - print('ver', name) - - + path = os.path.join(data_dir, name + ".bin") + if os.path.exists(path): + data_set = verification.load_bin(path, image_size) + ver_list.append(data_set) + ver_name_list.append(name) + print('ver', name) def ver_test(nbatch): - results = [] - for i in range(len(ver_list)): - acc1, std1, acc2, std2, xnorm, embeddings_list = verification.test(ver_list[i], model, args.batch_size, 10, None, None) - print('[%s][%d]XNorm: %f' % (ver_name_list[i], nbatch, xnorm)) - #print('[%s][%d]Accuracy: %1.5f+-%1.5f' % (ver_name_list[i], nbatch, acc1, std1)) - print('[%s][%d]Accuracy-Flip: %1.5f+-%1.5f' % (ver_name_list[i], nbatch, acc2, std2)) - results.append(acc2) - return results - - + results = [] + for i in range(len(ver_list)): + acc1, std1, acc2, std2, xnorm, embeddings_list = verification.test( + ver_list[i], model, args.batch_size, 10, None, None) + print('[%s][%d]XNorm: %f' % (ver_name_list[i], nbatch, xnorm)) + #print('[%s][%d]Accuracy: %1.5f+-%1.5f' % (ver_name_list[i], nbatch, acc1, std1)) + print('[%s][%d]Accuracy-Flip: %1.5f+-%1.5f' % + (ver_name_list[i], nbatch, acc2, std2)) + results.append(acc2) + return results highest_acc = [0.0, 0.0] #lfw and target #for i in range(len(ver_list)): @@ -287,92 +388,96 @@ def train_net(args): save_step = [0] lr_steps = [int(x) for x in args.lr_steps.split(',')] print('lr_steps', lr_steps) + def _batch_callback(param): - #global global_step - global_step[0]+=1 - mbatch = global_step[0] - for step in lr_steps: - if mbatch==step: - opt.lr *= 0.1 - print('lr change to', opt.lr) - break + #global global_step + global_step[0] += 1 + mbatch = global_step[0] + for step in lr_steps: + if mbatch == step: + opt.lr *= 0.1 + print('lr change to', opt.lr) + break - _cb(param) - if mbatch%1000==0: - print('lr-batch-epoch:',opt.lr,param.nbatch,param.epoch) + _cb(param) + if mbatch % 1000 == 0: + print('lr-batch-epoch:', opt.lr, param.nbatch, param.epoch) - if mbatch>=0 and mbatch%args.verbose==0: - acc_list = ver_test(mbatch) - save_step[0]+=1 - msave = save_step[0] - do_save = False - is_highest = False - if len(acc_list)>0: - #lfw_score = acc_list[0] - #if lfw_score>highest_acc[0]: - # highest_acc[0] = lfw_score - # if lfw_score>=0.998: - # do_save = True - score = sum(acc_list) - if acc_list[-1]>=highest_acc[-1]: - if acc_list[-1]>highest_acc[-1]: - is_highest = True - else: - if score>=highest_acc[0]: - is_highest = True - highest_acc[0] = score - highest_acc[-1] = acc_list[-1] - #if lfw_score>=0.99: - # do_save = True - if is_highest: - do_save = True - if args.ckpt==0: - do_save = False - elif args.ckpt==2: - do_save = True - elif args.ckpt==3: - msave = 1 + if mbatch >= 0 and mbatch % args.verbose == 0: + acc_list = ver_test(mbatch) + save_step[0] += 1 + msave = save_step[0] + do_save = False + is_highest = False + if len(acc_list) > 0: + #lfw_score = acc_list[0] + #if lfw_score>highest_acc[0]: + # highest_acc[0] = lfw_score + # if lfw_score>=0.998: + # do_save = True + score = sum(acc_list) + if acc_list[-1] >= highest_acc[-1]: + if acc_list[-1] > highest_acc[-1]: + is_highest = True + else: + if score >= highest_acc[0]: + is_highest = True + highest_acc[0] = score + highest_acc[-1] = acc_list[-1] + #if lfw_score>=0.99: + # do_save = True + if is_highest: + do_save = True + if args.ckpt == 0: + do_save = False + elif args.ckpt == 2: + do_save = True + elif args.ckpt == 3: + msave = 1 - if do_save: - print('saving', msave) - arg, aux = model.get_params() - if config.ckpt_embedding: - all_layers = model.symbol.get_internals() - _sym = all_layers['fc1_output'] - _arg = {} - for k in arg: - if not k.startswith('fc7'): - _arg[k] = arg[k] - mx.model.save_checkpoint(prefix, msave, _sym, _arg, aux) - else: - mx.model.save_checkpoint(prefix, msave, model.symbol, arg, aux) - print('[%d]Accuracy-Highest: %1.5f'%(mbatch, highest_acc[-1])) - if config.max_steps>0 and mbatch>config.max_steps: - sys.exit(0) + if do_save: + print('saving', msave) + arg, aux = model.get_params() + if config.ckpt_embedding: + all_layers = model.symbol.get_internals() + _sym = all_layers['fc1_output'] + _arg = {} + for k in arg: + if not k.startswith('fc7'): + _arg[k] = arg[k] + mx.model.save_checkpoint(prefix, msave, _sym, _arg, aux) + else: + mx.model.save_checkpoint(prefix, msave, model.symbol, arg, + aux) + print('[%d]Accuracy-Highest: %1.5f' % (mbatch, highest_acc[-1])) + if config.max_steps > 0 and mbatch > config.max_steps: + sys.exit(0) epoch_cb = None train_dataiter = mx.io.PrefetchingIter(train_dataiter) - model.fit(train_dataiter, - begin_epoch = begin_epoch, - num_epoch = 999999, - eval_data = val_dataiter, - eval_metric = eval_metrics, - kvstore = args.kvstore, - optimizer = opt, + model.fit( + train_dataiter, + begin_epoch=begin_epoch, + num_epoch=999999, + eval_data=val_dataiter, + eval_metric=eval_metrics, + kvstore=args.kvstore, + optimizer=opt, #optimizer_params = optimizer_params, - initializer = initializer, - arg_params = arg_params, - aux_params = aux_params, - allow_missing = True, - batch_end_callback = _batch_callback, - epoch_end_callback = epoch_cb ) + initializer=initializer, + arg_params=arg_params, + aux_params=aux_params, + allow_missing=True, + batch_end_callback=_batch_callback, + epoch_end_callback=epoch_cb) + def main(): global args args = parse_args() train_net(args) + if __name__ == '__main__': main() - diff --git a/recognition/ArcFace/train_parall.py b/recognition/ArcFace/train_parall.py index a400ac4..67cf389 100644 --- a/recognition/ArcFace/train_parall.py +++ b/recognition/ArcFace/train_parall.py @@ -1,4 +1,3 @@ - ''' @author: insightface ''' @@ -32,92 +31,155 @@ import fmnasnet import fdensenet import vargfacenet - logger = logging.getLogger() logger.setLevel(logging.INFO) - args = None - def parse_args(): - parser = argparse.ArgumentParser(description='Train parall face network') - # general - parser.add_argument('--dataset', default=default.dataset, help='dataset config') - parser.add_argument('--network', default=default.network, help='network config') - parser.add_argument('--loss', default=default.loss, help='loss config') - args, rest = parser.parse_known_args() - generate_config(args.network, args.dataset, args.loss) - parser.add_argument('--models-root', default=default.models_root, help='root directory to save model.') - parser.add_argument('--pretrained', default=default.pretrained, help='pretrained model to load') - parser.add_argument('--pretrained-epoch', type=int, default=default.pretrained_epoch, help='pretrained epoch to load') - parser.add_argument('--ckpt', type=int, default=default.ckpt, help='checkpoint saving option. 0: discard saving. 1: save when necessary. 2: always save') - parser.add_argument('--verbose', type=int, default=default.verbose, help='do verification testing and model saving every verbose batches') - parser.add_argument('--lr', type=float, default=default.lr, help='start learning rate') - parser.add_argument('--lr-steps', type=str, default=default.lr_steps, help='steps of lr changing') - parser.add_argument('--wd', type=float, default=default.wd, help='weight decay') - parser.add_argument('--mom', type=float, default=default.mom, help='momentum') - parser.add_argument('--frequent', type=int, default=default.frequent, help='') - parser.add_argument('--per-batch-size', type=int, default=default.per_batch_size, help='batch size in each context') - parser.add_argument('--kvstore', type=str, default=default.kvstore, help='kvstore setting') - parser.add_argument('--worker-id', type=int, default=0, help='worker id for dist training, starts from 0') - parser.add_argument('--extra-model-name', type=str, default='', help='extra model name') - args = parser.parse_args() - return args + parser = argparse.ArgumentParser(description='Train parall face network') + # general + parser.add_argument('--dataset', + default=default.dataset, + help='dataset config') + parser.add_argument('--network', + default=default.network, + help='network config') + parser.add_argument('--loss', default=default.loss, help='loss config') + args, rest = parser.parse_known_args() + generate_config(args.network, args.dataset, args.loss) + parser.add_argument('--models-root', + default=default.models_root, + help='root directory to save model.') + parser.add_argument('--pretrained', + default=default.pretrained, + help='pretrained model to load') + parser.add_argument('--pretrained-epoch', + type=int, + default=default.pretrained_epoch, + help='pretrained epoch to load') + parser.add_argument( + '--ckpt', + type=int, + default=default.ckpt, + help= + 'checkpoint saving option. 0: discard saving. 1: save when necessary. 2: always save' + ) + parser.add_argument( + '--verbose', + type=int, + default=default.verbose, + help='do verification testing and model saving every verbose batches') + parser.add_argument('--lr', + type=float, + default=default.lr, + help='start learning rate') + parser.add_argument('--lr-steps', + type=str, + default=default.lr_steps, + help='steps of lr changing') + parser.add_argument('--wd', + type=float, + default=default.wd, + help='weight decay') + parser.add_argument('--mom', + type=float, + default=default.mom, + help='momentum') + parser.add_argument('--frequent', + type=int, + default=default.frequent, + help='') + parser.add_argument('--per-batch-size', + type=int, + default=default.per_batch_size, + help='batch size in each context') + parser.add_argument('--kvstore', + type=str, + default=default.kvstore, + help='kvstore setting') + parser.add_argument('--worker-id', + type=int, + default=0, + help='worker id for dist training, starts from 0') + parser.add_argument('--extra-model-name', + type=str, + default='', + help='extra model name') + args = parser.parse_args() + return args -def get_symbol_embedding(): - embedding = eval(config.net_name).get_symbol() - all_label = mx.symbol.Variable('softmax_label') - #embedding = mx.symbol.BlockGrad(embedding) - all_label = mx.symbol.BlockGrad(all_label) - out_list = [embedding, all_label] - out = mx.symbol.Group(out_list) - return out +def get_symbol_embedding(embedding=None): + if embedding is None: + embedding = eval(config.net_name).get_symbol() + all_label = mx.symbol.Variable('softmax_label') + #embedding = mx.symbol.BlockGrad(embedding) + all_label = mx.symbol.BlockGrad(all_label) + out_list = [embedding, all_label] + out = mx.symbol.Group(out_list) + return out + def get_symbol_arcface(args): - embedding = mx.symbol.Variable('data') - all_label = mx.symbol.Variable('softmax_label') - gt_label = all_label - is_softmax = True - #print('call get_sym_arcface with', args, config) - _weight = mx.symbol.Variable("fc7_%d_weight"%args._ctxid, shape=(args.ctx_num_classes, config.emb_size), - lr_mult=config.fc7_lr_mult, wd_mult=config.fc7_wd_mult) - if config.loss_name=='softmax': #softmax - fc7 = mx.sym.FullyConnected(data=embedding, weight = _weight, no_bias = True, num_hidden=args.ctx_num_classes, name='fc7_%d'%args._ctxid) - elif config.loss_name=='margin_softmax': - _weight = mx.symbol.L2Normalization(_weight, mode='instance') - nembedding = mx.symbol.L2Normalization(embedding, mode='instance', name='fc1n_%d'%args._ctxid) - fc7 = mx.sym.FullyConnected(data=nembedding, weight = _weight, no_bias = True, num_hidden=args.ctx_num_classes, name='fc7_%d'%args._ctxid) - if config.loss_m1!=1.0 or config.loss_m2!=0.0 or config.loss_m3!=0.0: - gt_one_hot = mx.sym.one_hot(gt_label, depth = args.ctx_num_classes, on_value = 1.0, off_value = 0.0) - if config.loss_m1==1.0 and config.loss_m2==0.0: - _one_hot = gt_one_hot*config.loss_m3 - fc7 = fc7-_one_hot - else: - fc7_onehot = fc7 * gt_one_hot - cos_t = fc7_onehot - t = mx.sym.arccos(cos_t) - if config.loss_m1!=1.0: - t = t*config.loss_m1 - if config.loss_m2!=0.0: - t = t+config.loss_m2 - margin_cos = mx.sym.cos(t) - if config.loss_m3!=0.0: - margin_cos = margin_cos - config.loss_m3 - margin_fc7 = margin_cos - margin_fc7_onehot = margin_fc7 * gt_one_hot - diff = margin_fc7_onehot - fc7_onehot - fc7 = fc7+diff - fc7 = fc7*config.loss_s + embedding = mx.symbol.Variable('data') + all_label = mx.symbol.Variable('softmax_label') + gt_label = all_label + is_softmax = True + #print('call get_sym_arcface with', args, config) + _weight = mx.symbol.Variable("fc7_%d_weight" % args._ctxid, + shape=(args.ctx_num_classes, config.emb_size), + lr_mult=config.fc7_lr_mult, + wd_mult=config.fc7_wd_mult) + if config.loss_name == 'softmax': #softmax + fc7 = mx.sym.FullyConnected(data=embedding, + weight=_weight, + no_bias=True, + num_hidden=args.ctx_num_classes, + name='fc7_%d' % args._ctxid) + elif config.loss_name == 'margin_softmax': + _weight = mx.symbol.L2Normalization(_weight, mode='instance') + nembedding = mx.symbol.L2Normalization(embedding, + mode='instance', + name='fc1n_%d' % args._ctxid) + fc7 = mx.sym.FullyConnected(data=nembedding, + weight=_weight, + no_bias=True, + num_hidden=args.ctx_num_classes, + name='fc7_%d' % args._ctxid) + if config.loss_m1 != 1.0 or config.loss_m2 != 0.0 or config.loss_m3 != 0.0: + gt_one_hot = mx.sym.one_hot(gt_label, + depth=args.ctx_num_classes, + on_value=1.0, + off_value=0.0) + if config.loss_m1 == 1.0 and config.loss_m2 == 0.0: + _one_hot = gt_one_hot * config.loss_m3 + fc7 = fc7 - _one_hot + else: + fc7_onehot = fc7 * gt_one_hot + cos_t = fc7_onehot + t = mx.sym.arccos(cos_t) + if config.loss_m1 != 1.0: + t = t * config.loss_m1 + if config.loss_m2 != 0.0: + t = t + config.loss_m2 + margin_cos = mx.sym.cos(t) + if config.loss_m3 != 0.0: + margin_cos = margin_cos - config.loss_m3 + margin_fc7 = margin_cos + margin_fc7_onehot = margin_fc7 * gt_one_hot + diff = margin_fc7_onehot - fc7_onehot + fc7 = fc7 + diff + fc7 = fc7 * config.loss_s + + out_list = [] + out_list.append(fc7) + if config.loss_name == 'softmax': #softmax + out_list.append(gt_label) + out = mx.symbol.Group(out_list) + return out - out_list = [] - out_list.append(fc7) - if config.loss_name=='softmax': #softmax - out_list.append(gt_label) - out = mx.symbol.Group(out_list) - return out def train_net(args): #_seed = 727 @@ -126,26 +188,31 @@ def train_net(args): #mx.random.seed(_seed) ctx = [] cvd = os.environ['CUDA_VISIBLE_DEVICES'].strip() - if len(cvd)>0: - for i in range(len(cvd.split(','))): - ctx.append(mx.gpu(i)) - if len(ctx)==0: - ctx = [mx.cpu()] - print('use cpu') + if len(cvd) > 0: + for i in range(len(cvd.split(','))): + ctx.append(mx.gpu(i)) + if len(ctx) == 0: + ctx = [mx.cpu()] + print('use cpu') else: - print('gpu num:', len(ctx)) - if len(args.extra_model_name)==0: - prefix = os.path.join(args.models_root, '%s-%s-%s'%(args.network, args.loss, args.dataset), 'model') + print('gpu num:', len(ctx)) + if len(args.extra_model_name) == 0: + prefix = os.path.join( + args.models_root, + '%s-%s-%s' % (args.network, args.loss, args.dataset), 'model') else: - prefix = os.path.join(args.models_root, '%s-%s-%s-%s'%(args.network, args.loss, args.dataset, args.extra_model_name), 'model') + prefix = os.path.join( + args.models_root, '%s-%s-%s-%s' % + (args.network, args.loss, args.dataset, args.extra_model_name), + 'model') prefix_dir = os.path.dirname(prefix) print('prefix', prefix) if not os.path.exists(prefix_dir): - os.makedirs(prefix_dir) + os.makedirs(prefix_dir) args.ctx_num = len(ctx) - if args.per_batch_size==0: - args.per_batch_size = 128 - args.batch_size = args.per_batch_size*args.ctx_num + if args.per_batch_size == 0: + args.per_batch_size = 128 + args.batch_size = args.per_batch_size * args.ctx_num args.rescale_threshold = 0 args.image_channel = config.image_shape[2] config.batch_size = args.batch_size @@ -154,20 +221,20 @@ def train_net(args): path_imgrec = None path_imglist = None image_size = config.image_shape[0:2] - assert len(image_size)==2 - assert image_size[0]==image_size[1] + assert len(image_size) == 2 + assert image_size[0] == image_size[1] print('image_size', image_size) print('num_classes', config.num_classes) path_imgrec = os.path.join(data_dir, "train.rec") - data_shape = (args.image_channel,image_size[0],image_size[1]) + data_shape = (args.image_channel, image_size[0], image_size[1]) num_workers = config.num_workers global_num_ctx = num_workers * args.ctx_num - if config.num_classes%global_num_ctx==0: - args.ctx_num_classes = config.num_classes//global_num_ctx + if config.num_classes % global_num_ctx == 0: + args.ctx_num_classes = config.num_classes // global_num_ctx else: - args.ctx_num_classes = config.num_classes//global_num_ctx+1 + args.ctx_num_classes = config.num_classes // global_num_ctx + 1 args.local_num_classes = args.ctx_num_classes * args.ctx_num args.local_class_start = args.local_num_classes * args.worker_id @@ -189,82 +256,89 @@ def train_net(args): base_mom = args.mom arg_params = None aux_params = None - if len(args.pretrained)==0: - esym = get_symbol_embedding() - asym = get_symbol_arcface + if len(args.pretrained) == 0: + esym = get_symbol_embedding() + asym = get_symbol_arcface else: - #assert False - print('loading', args.pretrained, args.pretrained_epoch) - _, arg_params, aux_params = mx.model.load_checkpoint(args.pretrained, args.pretrained_epoch) - esym = get_symbol_embedding() - asym = get_symbol_arcface + #assert False + print('loading', args.pretrained, args.pretrained_epoch) + pretrain_esym, arg_params, aux_params = mx.model.load_checkpoint( + args.pretrained, args.pretrained_epoch) + esym = get_symbol_embedding(pretrain_esym) + asym = get_symbol_arcface if config.count_flops: - all_layers = esym.get_internals() - _sym = all_layers['fc1_output'] - FLOPs = flops_counter.count_flops(_sym, data=(1,3,image_size[0],image_size[1])) - _str = flops_counter.flops_str(FLOPs) - print('Network FLOPs: %s'%_str) + all_layers = esym.get_internals() + _sym = all_layers['fc1_output'] + FLOPs = flops_counter.count_flops(_sym, + data=(1, 3, image_size[0], + image_size[1])) + _str = flops_counter.flops_str(FLOPs) + print('Network FLOPs: %s' % _str) - if config.num_workers==1: - from parall_module_local_v1 import ParallModule + if config.num_workers == 1: + from parall_module_local_v1 import ParallModule else: - from parall_module_dist import ParallModule + from parall_module_dist import ParallModule model = ParallModule( - context = ctx, - symbol = esym, - data_names = ['data'], - label_names = ['softmax_label'], - asymbol = asym, - args = args, + context=ctx, + symbol=esym, + data_names=['data'], + label_names=['softmax_label'], + asymbol=asym, + args=args, ) val_dataiter = None train_dataiter = FaceImageIter( - batch_size = args.batch_size, - data_shape = data_shape, - path_imgrec = path_imgrec, - shuffle = True, - rand_mirror = config.data_rand_mirror, - mean = mean, - cutoff = config.data_cutoff, - color_jittering = config.data_color, - images_filter = config.data_images_filter, + batch_size=args.batch_size, + data_shape=data_shape, + path_imgrec=path_imgrec, + shuffle=True, + rand_mirror=config.data_rand_mirror, + mean=mean, + cutoff=config.data_cutoff, + color_jittering=config.data_color, + images_filter=config.data_images_filter, ) - - - if config.net_name=='fresnet' or config.net_name=='fmobilefacenet': - initializer = mx.init.Xavier(rnd_type='gaussian', factor_type="out", magnitude=2) #resnet style + if config.net_name == 'fresnet' or config.net_name == 'fmobilefacenet': + initializer = mx.init.Xavier(rnd_type='gaussian', + factor_type="out", + magnitude=2) #resnet style else: - initializer = mx.init.Xavier(rnd_type='uniform', factor_type="in", magnitude=2) + initializer = mx.init.Xavier(rnd_type='uniform', + factor_type="in", + magnitude=2) - _rescale = 1.0/args.batch_size - opt = optimizer.SGD(learning_rate=base_lr, momentum=base_mom, wd=base_wd, rescale_grad=_rescale) + _rescale = 1.0 / args.batch_size + opt = optimizer.SGD(learning_rate=base_lr, + momentum=base_mom, + wd=base_wd, + rescale_grad=_rescale) _cb = mx.callback.Speedometer(args.batch_size, args.frequent) - ver_list = [] ver_name_list = [] for name in config.val_targets: - path = os.path.join(data_dir,name+".bin") - if os.path.exists(path): - data_set = verification.load_bin(path, image_size) - ver_list.append(data_set) - ver_name_list.append(name) - print('ver', name) - + path = os.path.join(data_dir, name + ".bin") + if os.path.exists(path): + data_set = verification.load_bin(path, image_size) + ver_list.append(data_set) + ver_name_list.append(name) + print('ver', name) def ver_test(nbatch): - results = [] - for i in range(len(ver_list)): - acc1, std1, acc2, std2, xnorm, embeddings_list = verification.test(ver_list[i], model, args.batch_size, 10, None, None) - print('[%s][%d]XNorm: %f' % (ver_name_list[i], nbatch, xnorm)) - #print('[%s][%d]Accuracy: %1.5f+-%1.5f' % (ver_name_list[i], nbatch, acc1, std1)) - print('[%s][%d]Accuracy-Flip: %1.5f+-%1.5f' % (ver_name_list[i], nbatch, acc2, std2)) - results.append(acc2) - return results - + results = [] + for i in range(len(ver_list)): + acc1, std1, acc2, std2, xnorm, embeddings_list = verification.test( + ver_list[i], model, args.batch_size, 10, None, None) + print('[%s][%d]XNorm: %f' % (ver_name_list[i], nbatch, xnorm)) + #print('[%s][%d]Accuracy: %1.5f+-%1.5f' % (ver_name_list[i], nbatch, acc1, std1)) + print('[%s][%d]Accuracy-Flip: %1.5f+-%1.5f' % + (ver_name_list[i], nbatch, acc2, std2)) + results.append(acc2) + return results highest_acc = [0.0, 0.0] #lfw and target #for i in range(len(ver_list)): @@ -273,85 +347,88 @@ def train_net(args): save_step = [0] lr_steps = [int(x) for x in args.lr_steps.split(',')] print('lr_steps', lr_steps) + def _batch_callback(param): - #global global_step - global_step[0]+=1 - mbatch = global_step[0] - for step in lr_steps: - if mbatch==step: - opt.lr *= 0.1 - print('lr change to', opt.lr) - break + #global global_step + global_step[0] += 1 + mbatch = global_step[0] + for step in lr_steps: + if mbatch == step: + opt.lr *= 0.1 + print('lr change to', opt.lr) + break - _cb(param) - if mbatch%1000==0: - print('lr-batch-epoch:',opt.lr,param.nbatch,param.epoch) + _cb(param) + if mbatch % 1000 == 0: + print('lr-batch-epoch:', opt.lr, param.nbatch, param.epoch) - if mbatch>=0 and mbatch%args.verbose==0: - acc_list = ver_test(mbatch) - save_step[0]+=1 - msave = save_step[0] - do_save = False - is_highest = False - if len(acc_list)>0: - #lfw_score = acc_list[0] - #if lfw_score>highest_acc[0]: - # highest_acc[0] = lfw_score - # if lfw_score>=0.998: - # do_save = True - score = sum(acc_list) - if acc_list[-1]>=highest_acc[-1]: - if acc_list[-1]>highest_acc[-1]: - is_highest = True - else: - if score>=highest_acc[0]: - is_highest = True - highest_acc[0] = score - highest_acc[-1] = acc_list[-1] - #if lfw_score>=0.99: - # do_save = True - if is_highest: - do_save = True - if args.ckpt==0: - do_save = False - elif args.ckpt==2: - do_save = True - elif args.ckpt==3: - msave = 1 + if mbatch >= 0 and mbatch % args.verbose == 0: + acc_list = ver_test(mbatch) + save_step[0] += 1 + msave = save_step[0] + do_save = False + is_highest = False + if len(acc_list) > 0: + #lfw_score = acc_list[0] + #if lfw_score>highest_acc[0]: + # highest_acc[0] = lfw_score + # if lfw_score>=0.998: + # do_save = True + score = sum(acc_list) + if acc_list[-1] >= highest_acc[-1]: + if acc_list[-1] > highest_acc[-1]: + is_highest = True + else: + if score >= highest_acc[0]: + is_highest = True + highest_acc[0] = score + highest_acc[-1] = acc_list[-1] + #if lfw_score>=0.99: + # do_save = True + if is_highest: + do_save = True + if args.ckpt == 0: + do_save = False + elif args.ckpt == 2: + do_save = True + elif args.ckpt == 3: + msave = 1 - if do_save: - print('saving', msave) - arg, aux = model.get_export_params() - all_layers = model.symbol.get_internals() - _sym = all_layers['fc1_output'] - mx.model.save_checkpoint(prefix, msave, _sym, arg, aux) - print('[%d]Accuracy-Highest: %1.5f'%(mbatch, highest_acc[-1])) - if config.max_steps>0 and mbatch>config.max_steps: - sys.exit(0) + if do_save: + print('saving', msave) + arg, aux = model.get_export_params() + all_layers = model.symbol.get_internals() + _sym = all_layers['fc1_output'] + mx.model.save_checkpoint(prefix, msave, _sym, arg, aux) + print('[%d]Accuracy-Highest: %1.5f' % (mbatch, highest_acc[-1])) + if config.max_steps > 0 and mbatch > config.max_steps: + sys.exit(0) epoch_cb = None train_dataiter = mx.io.PrefetchingIter(train_dataiter) - model.fit(train_dataiter, - begin_epoch = begin_epoch, - num_epoch = 999999, - eval_data = val_dataiter, + model.fit( + train_dataiter, + begin_epoch=begin_epoch, + num_epoch=999999, + eval_data=val_dataiter, #eval_metric = eval_metrics, - kvstore = args.kvstore, - optimizer = opt, + kvstore=args.kvstore, + optimizer=opt, #optimizer_params = optimizer_params, - initializer = initializer, - arg_params = arg_params, - aux_params = aux_params, - allow_missing = True, - batch_end_callback = _batch_callback, - epoch_end_callback = epoch_cb ) + initializer=initializer, + arg_params=arg_params, + aux_params=aux_params, + allow_missing=True, + batch_end_callback=_batch_callback, + epoch_end_callback=epoch_cb) + def main(): global args args = parse_args() train_net(args) + if __name__ == '__main__': main() - diff --git a/recognition/ArcFace/triplet_image_iter.py b/recognition/ArcFace/triplet_image_iter.py index 6c8a43e..1df39ac 100644 --- a/recognition/ArcFace/triplet_image_iter.py +++ b/recognition/ArcFace/triplet_image_iter.py @@ -26,26 +26,32 @@ import face_preprocess logger = logging.getLogger() -class FaceImageIter(io.DataIter): - def __init__(self, batch_size, data_shape, - path_imgrec = None, - shuffle=False, aug_list=None, - rand_mirror = False, cutoff = 0, - ctx_num = 0, images_per_identity = 0, - triplet_params = None, - mx_model = None, - data_name='data', label_name='softmax_label', **kwargs): +class FaceImageIter(io.DataIter): + def __init__(self, + batch_size, + data_shape, + path_imgrec=None, + shuffle=False, + aug_list=None, + rand_mirror=False, + cutoff=0, + ctx_num=0, + images_per_identity=0, + triplet_params=None, + mx_model=None, + data_name='data', + label_name='softmax_label', + **kwargs): super(FaceImageIter, self).__init__() assert path_imgrec assert shuffle - logging.info('loading recordio %s...', - path_imgrec) - path_imgidx = path_imgrec[0:-4]+".idx" + logging.info('loading recordio %s...', path_imgrec) + path_imgidx = path_imgrec[0:-4] + ".idx" self.imgrec = recordio.MXIndexedRecordIO(path_imgidx, path_imgrec, 'r') # pylint: disable=redefined-variable-type s = self.imgrec.read_idx(0) header, _ = recordio.unpack(s) - assert header.flag>0 + assert header.flag > 0 print('header0 label', header.label) self.header0 = (int(header.label[0]), int(header.label[1])) #assert(header.flag==1) @@ -53,461 +59,473 @@ class FaceImageIter(io.DataIter): self.id2range = {} self.seq_identity = range(int(header.label[0]), int(header.label[1])) for identity in self.seq_identity: - s = self.imgrec.read_idx(identity) - header, _ = recordio.unpack(s) - a,b = int(header.label[0]), int(header.label[1]) - self.id2range[identity] = (a,b) + s = self.imgrec.read_idx(identity) + header, _ = recordio.unpack(s) + a, b = int(header.label[0]), int(header.label[1]) + self.id2range[identity] = (a, b) print('id2range', len(self.id2range)) self.seq = self.imgidx print(len(self.seq)) self.check_data_shape(data_shape) - self.provide_data = [(data_name, (batch_size,) + data_shape)] + self.provide_data = [(data_name, (batch_size, ) + data_shape)] self.batch_size = batch_size self.data_shape = data_shape self.shuffle = shuffle - self.image_size = '%d,%d'%(data_shape[1],data_shape[2]) + self.image_size = '%d,%d' % (data_shape[1], data_shape[2]) self.rand_mirror = rand_mirror print('rand_mirror', rand_mirror) self.cutoff = cutoff #self.cast_aug = mx.image.CastAug() #self.color_aug = mx.image.ColorJitterAug(0.4, 0.4, 0.4) - self.ctx_num = ctx_num - self.per_batch_size = int(self.batch_size/self.ctx_num) + self.ctx_num = ctx_num + self.per_batch_size = int(self.batch_size / self.ctx_num) self.images_per_identity = images_per_identity - if self.images_per_identity>0: - self.identities = int(self.per_batch_size/self.images_per_identity) - self.per_identities = self.identities - self.repeat = 3000000.0/(self.images_per_identity*len(self.id2range)) - self.repeat = int(self.repeat) - print(self.images_per_identity, self.identities, self.repeat) + if self.images_per_identity > 0: + self.identities = int(self.per_batch_size / + self.images_per_identity) + self.per_identities = self.identities + self.repeat = 3000000.0 / (self.images_per_identity * + len(self.id2range)) + self.repeat = int(self.repeat) + print(self.images_per_identity, self.identities, self.repeat) self.mx_model = mx_model self.triplet_params = triplet_params self.triplet_mode = False #self.provide_label = None - self.provide_label = [(label_name, (batch_size,))] + self.provide_label = [(label_name, (batch_size, ))] if self.triplet_params is not None: - assert self.images_per_identity>0 - assert self.mx_model is not None - self.triplet_bag_size = self.triplet_params[0] - self.triplet_alpha = self.triplet_params[1] - self.triplet_max_ap = self.triplet_params[2] - assert self.triplet_bag_size>0 - assert self.triplet_alpha>=0.0 - assert self.triplet_alpha<=1.0 - self.triplet_mode = True - self.triplet_cur = 0 - self.triplet_seq = [] - self.triplet_reset() - self.seq_min_size = self.batch_size*2 + assert self.images_per_identity > 0 + assert self.mx_model is not None + self.triplet_bag_size = self.triplet_params[0] + self.triplet_alpha = self.triplet_params[1] + self.triplet_max_ap = self.triplet_params[2] + assert self.triplet_bag_size > 0 + assert self.triplet_alpha >= 0.0 + assert self.triplet_alpha <= 1.0 + self.triplet_mode = True + self.triplet_cur = 0 + self.triplet_seq = [] + self.triplet_reset() + self.seq_min_size = self.batch_size * 2 self.cur = 0 self.nbatch = 0 self.is_init = False self.times = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0] #self.reset() - - def pairwise_dists(self, embeddings): - nd_embedding_list = [] - for i in range(self.ctx_num): - nd_embedding = mx.nd.array(embeddings, mx.gpu(i)) - nd_embedding_list.append(nd_embedding) - nd_pdists = [] - pdists = [] - for idx in range(embeddings.shape[0]): - emb_idx = idx%self.ctx_num - nd_embedding = nd_embedding_list[emb_idx] - a_embedding = nd_embedding[idx] - body = mx.nd.broadcast_sub(a_embedding, nd_embedding) - body = body*body - body = mx.nd.sum_axis(body, axis=1) - nd_pdists.append(body) - if len(nd_pdists)==self.ctx_num or idx==embeddings.shape[0]-1: - for x in nd_pdists: - pdists.append(x.asnumpy()) - nd_pdists = [] - return pdists + nd_embedding_list = [] + for i in range(self.ctx_num): + nd_embedding = mx.nd.array(embeddings, mx.gpu(i)) + nd_embedding_list.append(nd_embedding) + nd_pdists = [] + pdists = [] + for idx in range(embeddings.shape[0]): + emb_idx = idx % self.ctx_num + nd_embedding = nd_embedding_list[emb_idx] + a_embedding = nd_embedding[idx] + body = mx.nd.broadcast_sub(a_embedding, nd_embedding) + body = body * body + body = mx.nd.sum_axis(body, axis=1) + nd_pdists.append(body) + if len(nd_pdists + ) == self.ctx_num or idx == embeddings.shape[0] - 1: + for x in nd_pdists: + pdists.append(x.asnumpy()) + nd_pdists = [] + return pdists def pick_triplets(self, embeddings, nrof_images_per_class): - emb_start_idx = 0 - triplets = [] - people_per_batch = len(nrof_images_per_class) - #self.time_reset() - pdists = self.pairwise_dists(embeddings) - #self.times[3] += self.time_elapsed() + emb_start_idx = 0 + triplets = [] + people_per_batch = len(nrof_images_per_class) + #self.time_reset() + pdists = self.pairwise_dists(embeddings) + #self.times[3] += self.time_elapsed() - for i in range(people_per_batch): - nrof_images = int(nrof_images_per_class[i]) - for j in range(1,nrof_images): - #self.time_reset() - a_idx = emb_start_idx + j - 1 - #neg_dists_sqr = np.sum(np.square(embeddings[a_idx] - embeddings), 1) - neg_dists_sqr = pdists[a_idx] - #self.times[3] += self.time_elapsed() + for i in range(people_per_batch): + nrof_images = int(nrof_images_per_class[i]) + for j in range(1, nrof_images): + #self.time_reset() + a_idx = emb_start_idx + j - 1 + #neg_dists_sqr = np.sum(np.square(embeddings[a_idx] - embeddings), 1) + neg_dists_sqr = pdists[a_idx] + #self.times[3] += self.time_elapsed() - for pair in range(j, nrof_images): # For every possible positive pair. - p_idx = emb_start_idx + pair - #self.time_reset() - pos_dist_sqr = np.sum(np.square(embeddings[a_idx]-embeddings[p_idx])) - #self.times[4] += self.time_elapsed() - #self.time_reset() - neg_dists_sqr[emb_start_idx:emb_start_idx+nrof_images] = np.NaN - if self.triplet_max_ap>0.0: - if pos_dist_sqr>self.triplet_max_ap: - continue - all_neg = np.where(np.logical_and(neg_dists_sqr-pos_dist_sqr0: - rnd_idx = np.random.randint(nrof_random_negs) - n_idx = all_neg[rnd_idx] - triplets.append( (a_idx, p_idx, n_idx) ) - emb_start_idx += nrof_images - np.random.shuffle(triplets) - return triplets + for pair in range( + j, nrof_images): # For every possible positive pair. + p_idx = emb_start_idx + pair + #self.time_reset() + pos_dist_sqr = np.sum( + np.square(embeddings[a_idx] - embeddings[p_idx])) + #self.times[4] += self.time_elapsed() + #self.time_reset() + neg_dists_sqr[emb_start_idx:emb_start_idx + + nrof_images] = np.NaN + if self.triplet_max_ap > 0.0: + if pos_dist_sqr > self.triplet_max_ap: + continue + all_neg = np.where( + np.logical_and( + neg_dists_sqr - pos_dist_sqr < self.triplet_alpha, + pos_dist_sqr < + neg_dists_sqr))[0] # FaceNet selection + #self.times[5] += self.time_elapsed() + #self.time_reset() + #all_neg = np.where(neg_dists_sqr-pos_dist_sqr 0: + rnd_idx = np.random.randint(nrof_random_negs) + n_idx = all_neg[rnd_idx] + triplets.append((a_idx, p_idx, n_idx)) + emb_start_idx += nrof_images + np.random.shuffle(triplets) + return triplets def triplet_reset(self): - #reset self.oseq by identities seq - self.triplet_cur = 0 - ids = [] - for k in self.id2range: - ids.append(k) - random.shuffle(ids) - self.triplet_seq = [] - for _id in ids: - v = self.id2range[_id] - _list = range(*v) - random.shuffle(_list) - if len(_list)>self.images_per_identity: - _list = _list[0:self.images_per_identity] - self.triplet_seq += _list - print('triplet_seq', len(self.triplet_seq)) - assert len(self.triplet_seq)>=self.triplet_bag_size + #reset self.oseq by identities seq + self.triplet_cur = 0 + ids = [] + for k in self.id2range: + ids.append(k) + random.shuffle(ids) + self.triplet_seq = [] + for _id in ids: + v = self.id2range[_id] + _list = range(*v) + random.shuffle(_list) + if len(_list) > self.images_per_identity: + _list = _list[0:self.images_per_identity] + self.triplet_seq += _list + print('triplet_seq', len(self.triplet_seq)) + assert len(self.triplet_seq) >= self.triplet_bag_size def time_reset(self): - self.time_now = datetime.datetime.now() + self.time_now = datetime.datetime.now() def time_elapsed(self): - time_now = datetime.datetime.now() - diff = time_now - self.time_now - return diff.total_seconds() - + time_now = datetime.datetime.now() + diff = time_now - self.time_now + return diff.total_seconds() def select_triplets(self): - self.seq = [] - while len(self.seq)len(self.triplet_seq): - self.triplet_reset() - #bag_size = min(bag_size, len(self.triplet_seq)) - print('eval %d images..'%bag_size, self.triplet_cur) - self.times[0] += self.time_elapsed() - self.time_reset() - #print(data.shape) - data = nd.zeros( self.provide_data[0][1] ) - label = None - if self.provide_label is not None: - label = nd.zeros( self.provide_label[0][1] ) - ba = 0 - while True: - bb = min(ba+batch_size, bag_size) - if ba>=bb: - break - _count = bb-ba - #data = nd.zeros( (_count,)+self.data_shape ) - #_batch = self.data_iter.next() - #_data = _batch.data[0].asnumpy() - #print(_data.shape) - #_label = _batch.label[0].asnumpy() - #data[ba:bb,:,:,:] = _data - #label[ba:bb] = _label - for i in range(ba, bb): - #print(ba, bb, self.triplet_cur, i, len(self.triplet_seq)) - _idx = self.triplet_seq[i+self.triplet_cur] - s = self.imgrec.read_idx(_idx) - header, img = recordio.unpack(s) - img = self.imdecode(img) - data[i-ba][:] = self.postprocess_data(img) - _label = header.label - if not isinstance(_label, numbers.Number): - _label = _label[0] - if label is not None: - label[i-ba][:] = _label - tag.append( ( int(_label), _idx) ) - #idx[i] = _idx + self.seq = [] + while len(self.seq) < self.seq_min_size: + self.time_reset() + embeddings = None + bag_size = self.triplet_bag_size + batch_size = self.batch_size + #data = np.zeros( (bag_size,)+self.data_shape ) + #label = np.zeros( (bag_size,) ) + tag = [] + #idx = np.zeros( (bag_size,) ) + print('eval %d images..' % bag_size, self.triplet_cur) + print('triplet time stat', self.times) + if self.triplet_cur + bag_size > len(self.triplet_seq): + self.triplet_reset() + #bag_size = min(bag_size, len(self.triplet_seq)) + print('eval %d images..' % bag_size, self.triplet_cur) + self.times[0] += self.time_elapsed() + self.time_reset() + #print(data.shape) + data = nd.zeros(self.provide_data[0][1]) + label = None + if self.provide_label is not None: + label = nd.zeros(self.provide_label[0][1]) + ba = 0 + while True: + bb = min(ba + batch_size, bag_size) + if ba >= bb: + break + _count = bb - ba + #data = nd.zeros( (_count,)+self.data_shape ) + #_batch = self.data_iter.next() + #_data = _batch.data[0].asnumpy() + #print(_data.shape) + #_label = _batch.label[0].asnumpy() + #data[ba:bb,:,:,:] = _data + #label[ba:bb] = _label + for i in range(ba, bb): + #print(ba, bb, self.triplet_cur, i, len(self.triplet_seq)) + _idx = self.triplet_seq[i + self.triplet_cur] + s = self.imgrec.read_idx(_idx) + header, img = recordio.unpack(s) + img = self.imdecode(img) + data[i - ba][:] = self.postprocess_data(img) + _label = header.label + if not isinstance(_label, numbers.Number): + _label = _label[0] + if label is not None: + label[i - ba][:] = _label + tag.append((int(_label), _idx)) + #idx[i] = _idx - db = mx.io.DataBatch(data=(data,)) - self.mx_model.forward(db, is_train=False) - net_out = self.mx_model.get_outputs() - #print('eval for selecting triplets',ba,bb) - #print(net_out) - #print(len(net_out)) - #print(net_out[0].asnumpy()) - net_out = net_out[0].asnumpy() - #print(net_out) - #print('net_out', net_out.shape) - if embeddings is None: - embeddings = np.zeros( (bag_size, net_out.shape[1])) - embeddings[ba:bb,:] = net_out - ba = bb - assert len(tag)==bag_size - self.triplet_cur+=bag_size - embeddings = sklearn.preprocessing.normalize(embeddings) - self.times[1] += self.time_elapsed() - self.time_reset() - nrof_images_per_class = [1] - for i in range(1, bag_size): - if tag[i][0]==tag[i-1][0]: - nrof_images_per_class[-1]+=1 - else: - nrof_images_per_class.append(1) - - triplets = self.pick_triplets(embeddings, nrof_images_per_class) # shape=(T,3) - print('found triplets', len(triplets)) - ba = 0 - while True: - bb = ba+self.per_batch_size//3 - if bb>len(triplets): - break - _triplets = triplets[ba:bb] - for i in range(3): - for triplet in _triplets: - _pos = triplet[i] - _idx = tag[_pos][1] - self.seq.append(_idx) - ba = bb - self.times[2] += self.time_elapsed() + db = mx.io.DataBatch(data=(data, )) + self.mx_model.forward(db, is_train=False) + net_out = self.mx_model.get_outputs() + #print('eval for selecting triplets',ba,bb) + #print(net_out) + #print(len(net_out)) + #print(net_out[0].asnumpy()) + net_out = net_out[0].asnumpy() + #print(net_out) + #print('net_out', net_out.shape) + if embeddings is None: + embeddings = np.zeros((bag_size, net_out.shape[1])) + embeddings[ba:bb, :] = net_out + ba = bb + assert len(tag) == bag_size + self.triplet_cur += bag_size + embeddings = sklearn.preprocessing.normalize(embeddings) + self.times[1] += self.time_elapsed() + self.time_reset() + nrof_images_per_class = [1] + for i in range(1, bag_size): + if tag[i][0] == tag[i - 1][0]: + nrof_images_per_class[-1] += 1 + else: + nrof_images_per_class.append(1) + + triplets = self.pick_triplets(embeddings, + nrof_images_per_class) # shape=(T,3) + print('found triplets', len(triplets)) + ba = 0 + while True: + bb = ba + self.per_batch_size // 3 + if bb > len(triplets): + break + _triplets = triplets[ba:bb] + for i in range(3): + for triplet in _triplets: + _pos = triplet[i] + _idx = tag[_pos][1] + self.seq.append(_idx) + ba = bb + self.times[2] += self.time_elapsed() def hard_mining_reset(self): - #import faiss - from annoy import AnnoyIndex - data = nd.zeros( self.provide_data[0][1] ) - label = nd.zeros( self.provide_label[0][1] ) - #label = np.zeros( self.provide_label[0][1] ) - X = None - ba = 0 - batch_num = 0 - while ba0: - if self.triplet_mode: - self.select_triplets() - elif not self.hard_mining: - self.seq = [] - idlist = [] - for _id in self.id2range: - v = self.id2range[_id] - idlist.append((_id,range(*v))) - for r in range(self.repeat): - if r%10==0: - print('repeat', r) - if self.shuffle: - random.shuffle(idlist) - for item in idlist: - _id = item[0] - _list = item[1] - #random.shuffle(_list) - if len(_list) 0: + if self.triplet_mode: + self.select_triplets() + elif not self.hard_mining: + self.seq = [] + idlist = [] + for _id in self.id2range: + v = self.id2range[_id] + idlist.append((_id, range(*v))) + for r in range(self.repeat): + if r % 10 == 0: + print('repeat', r) + if self.shuffle: + random.shuffle(idlist) + for item in idlist: + _id = item[0] + _list = item[1] + #random.shuffle(_list) + if len(_list) < self.images_per_identity: + random.shuffle(_list) + else: + _list = np.random.choice(_list, + self.images_per_identity, + replace=False) + for i in range(self.images_per_identity): + _idx = _list[i % len(_list)] + self.seq.append(_idx) + else: + self.hard_mining_reset() + print('seq len', len(self.seq)) else: - if self.shuffle: - random.shuffle(self.seq) + if self.shuffle: + random.shuffle(self.seq) if self.seq is None and self.imgrec is not None: self.imgrec.reset() def num_samples(self): - return len(self.seq) + return len(self.seq) def next_sample(self): - while True: - if self.cur >= len(self.seq): - raise StopIteration - idx = self.seq[self.cur] - self.cur += 1 - s = self.imgrec.read_idx(idx) - header, img = recordio.unpack(s) - label = header.label - if not isinstance(label, numbers.Number): - label = label[0] - return label, img, None, None + while True: + if self.cur >= len(self.seq): + raise StopIteration + idx = self.seq[self.cur] + self.cur += 1 + s = self.imgrec.read_idx(idx) + header, img = recordio.unpack(s) + label = header.label + if not isinstance(label, numbers.Number): + label = label[0] + return label, img, None, None def brightness_aug(self, src, x): - alpha = 1.0 + random.uniform(-x, x) - src *= alpha - return src + alpha = 1.0 + random.uniform(-x, x) + src *= alpha + return src def contrast_aug(self, src, x): - alpha = 1.0 + random.uniform(-x, x) - coef = np.array([[[0.299, 0.587, 0.114]]]) - gray = src * coef - gray = (3.0 * (1.0 - alpha) / gray.size) * np.sum(gray) - src *= alpha - src += gray - return src + alpha = 1.0 + random.uniform(-x, x) + coef = np.array([[[0.299, 0.587, 0.114]]]) + gray = src * coef + gray = (3.0 * (1.0 - alpha) / gray.size) * np.sum(gray) + src *= alpha + src += gray + return src def saturation_aug(self, src, x): - alpha = 1.0 + random.uniform(-x, x) - coef = np.array([[[0.299, 0.587, 0.114]]]) - gray = src * coef - gray = np.sum(gray, axis=2, keepdims=True) - gray *= (1.0 - alpha) - src *= alpha - src += gray - return src + alpha = 1.0 + random.uniform(-x, x) + coef = np.array([[[0.299, 0.587, 0.114]]]) + gray = src * coef + gray = np.sum(gray, axis=2, keepdims=True) + gray *= (1.0 - alpha) + src *= alpha + src += gray + return src def color_aug(self, img, x): - augs = [self.brightness_aug, self.contrast_aug, self.saturation_aug] - random.shuffle(augs) - for aug in augs: - #print(img.shape) - img = aug(img, x) - #print(img.shape) - return img + augs = [self.brightness_aug, self.contrast_aug, self.saturation_aug] + random.shuffle(augs) + for aug in augs: + #print(img.shape) + img = aug(img, x) + #print(img.shape) + return img def mirror_aug(self, img): - _rd = random.randint(0,1) - if _rd==1: - for c in range(img.shape[2]): - img[:,:,c] = np.fliplr(img[:,:,c]) - return img - + _rd = random.randint(0, 1) + if _rd == 1: + for c in range(img.shape[2]): + img[:, :, c] = np.fliplr(img[:, :, c]) + return img def next(self): if not self.is_init: - self.reset() - self.is_init = True + self.reset() + self.is_init = True """Returns the next batch of data.""" #print('in next', self.cur, self.labelcur) - self.nbatch+=1 + self.nbatch += 1 batch_size = self.batch_size c, h, w = self.data_shape batch_data = nd.empty((batch_size, c, h, w)) if self.provide_label is not None: - batch_label = nd.empty(self.provide_label[0][1]) + batch_label = nd.empty(self.provide_label[0][1]) i = 0 try: while i < batch_size: label, s, bbox, landmark = self.next_sample() _data = self.imdecode(s) if self.rand_mirror: - _rd = random.randint(0,1) - if _rd==1: - _data = mx.ndarray.flip(data=_data, axis=1) - if self.cutoff>0: - centerh = random.randint(0, _data.shape[0]-1) - centerw = random.randint(0, _data.shape[1]-1) - half = self.cutoff//2 - starth = max(0, centerh-half) - endh = min(_data.shape[0], centerh+half) - startw = max(0, centerw-half) - endw = min(_data.shape[1], centerw+half) - _data = _data.astype('float32') - #print(starth, endh, startw, endw, _data.shape) - _data[starth:endh, startw:endw, :] = 127.5 + _rd = random.randint(0, 1) + if _rd == 1: + _data = mx.ndarray.flip(data=_data, axis=1) + if self.cutoff > 0: + centerh = random.randint(0, _data.shape[0] - 1) + centerw = random.randint(0, _data.shape[1] - 1) + half = self.cutoff // 2 + starth = max(0, centerh - half) + endh = min(_data.shape[0], centerh + half) + startw = max(0, centerw - half) + endw = min(_data.shape[1], centerw + half) + _data = _data.astype('float32') + #print(starth, endh, startw, endw, _data.shape) + _data[starth:endh, startw:endw, :] = 127.5 #_npdata = _data.asnumpy() #if landmark is not None: # _npdata = face_preprocess.preprocess(_npdata, bbox = bbox, landmark=landmark, image_size=self.image_size) @@ -534,24 +552,26 @@ class FaceImageIter(io.DataIter): #print(datum.shape) batch_data[i][:] = self.postprocess_data(datum) if self.provide_label is not None: - batch_label[i][:] = label + batch_label[i][:] = label i += 1 except StopIteration: - if i0 - self.provide_data = iter_list[0].provide_data - self.provide_label = iter_list[0].provide_label - self.iter_list = iter_list - self.cur_iter = None + def __init__(self, iter_list): + assert len(iter_list) > 0 + self.provide_data = iter_list[0].provide_data + self.provide_label = iter_list[0].provide_label + self.iter_list = iter_list + self.cur_iter = None - def reset(self): - self.cur_iter.reset() - - def next(self): - self.cur_iter = random.choice(self.iter_list) - while True: - try: - ret = self.cur_iter.next() - except StopIteration: + def reset(self): self.cur_iter.reset() - continue - return ret - + def next(self): + self.cur_iter = random.choice(self.iter_list) + while True: + try: + ret = self.cur_iter.next() + except StopIteration: + self.cur_iter.reset() + continue + return ret diff --git a/recognition/ArcFace/verification.py b/recognition/ArcFace/verification.py index 1b1c31c..b8f90a6 100644 --- a/recognition/ArcFace/verification.py +++ b/recognition/ArcFace/verification.py @@ -2,19 +2,19 @@ """ # MIT License -# +# # Copyright (c) 2016 David Sandberg -# +# # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: -# +# # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. -# +# # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE @@ -45,110 +45,131 @@ from mxnet import ndarray as nd class LFold: - def __init__(self, n_splits = 2, shuffle = False): - self.n_splits = n_splits - if self.n_splits>1: - self.k_fold = KFold(n_splits = n_splits, shuffle = shuffle) + def __init__(self, n_splits=2, shuffle=False): + self.n_splits = n_splits + if self.n_splits > 1: + self.k_fold = KFold(n_splits=n_splits, shuffle=shuffle) - def split(self, indices): - if self.n_splits>1: - return self.k_fold.split(indices) - else: - return [(indices, indices)] + def split(self, indices): + if self.n_splits > 1: + return self.k_fold.split(indices) + else: + return [(indices, indices)] -def calculate_roc(thresholds, embeddings1, embeddings2, actual_issame, nrof_folds=10, pca = 0): - assert(embeddings1.shape[0] == embeddings2.shape[0]) - assert(embeddings1.shape[1] == embeddings2.shape[1]) +def calculate_roc(thresholds, + embeddings1, + embeddings2, + actual_issame, + nrof_folds=10, + pca=0): + assert (embeddings1.shape[0] == embeddings2.shape[0]) + assert (embeddings1.shape[1] == embeddings2.shape[1]) nrof_pairs = min(len(actual_issame), embeddings1.shape[0]) nrof_thresholds = len(thresholds) k_fold = LFold(n_splits=nrof_folds, shuffle=False) - - tprs = np.zeros((nrof_folds,nrof_thresholds)) - fprs = np.zeros((nrof_folds,nrof_thresholds)) + + tprs = np.zeros((nrof_folds, nrof_thresholds)) + fprs = np.zeros((nrof_folds, nrof_thresholds)) accuracy = np.zeros((nrof_folds)) indices = np.arange(nrof_pairs) #print('pca', pca) - - if pca==0: - diff = np.subtract(embeddings1, embeddings2) - dist = np.sum(np.square(diff),1) - + + if pca == 0: + diff = np.subtract(embeddings1, embeddings2) + dist = np.sum(np.square(diff), 1) + for fold_idx, (train_set, test_set) in enumerate(k_fold.split(indices)): #print('train_set', train_set) #print('test_set', test_set) - if pca>0: - print('doing pca on', fold_idx) - embed1_train = embeddings1[train_set] - embed2_train = embeddings2[train_set] - _embed_train = np.concatenate( (embed1_train, embed2_train), axis=0 ) - #print(_embed_train.shape) - pca_model = PCA(n_components=pca) - pca_model.fit(_embed_train) - embed1 = pca_model.transform(embeddings1) - embed2 = pca_model.transform(embeddings2) - embed1 = sklearn.preprocessing.normalize(embed1) - embed2 = sklearn.preprocessing.normalize(embed2) - #print(embed1.shape, embed2.shape) - diff = np.subtract(embed1, embed2) - dist = np.sum(np.square(diff),1) - + if pca > 0: + print('doing pca on', fold_idx) + embed1_train = embeddings1[train_set] + embed2_train = embeddings2[train_set] + _embed_train = np.concatenate((embed1_train, embed2_train), axis=0) + #print(_embed_train.shape) + pca_model = PCA(n_components=pca) + pca_model.fit(_embed_train) + embed1 = pca_model.transform(embeddings1) + embed2 = pca_model.transform(embeddings2) + embed1 = sklearn.preprocessing.normalize(embed1) + embed2 = sklearn.preprocessing.normalize(embed2) + #print(embed1.shape, embed2.shape) + diff = np.subtract(embed1, embed2) + dist = np.sum(np.square(diff), 1) + # Find the best threshold for the fold acc_train = np.zeros((nrof_thresholds)) for threshold_idx, threshold in enumerate(thresholds): - _, _, acc_train[threshold_idx] = calculate_accuracy(threshold, dist[train_set], actual_issame[train_set]) + _, _, acc_train[threshold_idx] = calculate_accuracy( + threshold, dist[train_set], actual_issame[train_set]) best_threshold_index = np.argmax(acc_train) #print('threshold', thresholds[best_threshold_index]) for threshold_idx, threshold in enumerate(thresholds): - tprs[fold_idx,threshold_idx], fprs[fold_idx,threshold_idx], _ = calculate_accuracy(threshold, dist[test_set], actual_issame[test_set]) - _, _, accuracy[fold_idx] = calculate_accuracy(thresholds[best_threshold_index], dist[test_set], actual_issame[test_set]) - - tpr = np.mean(tprs,0) - fpr = np.mean(fprs,0) + tprs[fold_idx, + threshold_idx], fprs[fold_idx, + threshold_idx], _ = calculate_accuracy( + threshold, dist[test_set], + actual_issame[test_set]) + _, _, accuracy[fold_idx] = calculate_accuracy( + thresholds[best_threshold_index], dist[test_set], + actual_issame[test_set]) + + tpr = np.mean(tprs, 0) + fpr = np.mean(fprs, 0) return tpr, fpr, accuracy + def calculate_accuracy(threshold, dist, actual_issame): predict_issame = np.less(dist, threshold) tp = np.sum(np.logical_and(predict_issame, actual_issame)) fp = np.sum(np.logical_and(predict_issame, np.logical_not(actual_issame))) - tn = np.sum(np.logical_and(np.logical_not(predict_issame), np.logical_not(actual_issame))) + tn = np.sum( + np.logical_and(np.logical_not(predict_issame), + np.logical_not(actual_issame))) fn = np.sum(np.logical_and(np.logical_not(predict_issame), actual_issame)) - - tpr = 0 if (tp+fn==0) else float(tp) / float(tp+fn) - fpr = 0 if (fp+tn==0) else float(fp) / float(fp+tn) - acc = float(tp+tn)/dist.size + + tpr = 0 if (tp + fn == 0) else float(tp) / float(tp + fn) + fpr = 0 if (fp + tn == 0) else float(fp) / float(fp + tn) + acc = float(tp + tn) / dist.size return tpr, fpr, acc - -def calculate_val(thresholds, embeddings1, embeddings2, actual_issame, far_target, nrof_folds=10): - assert(embeddings1.shape[0] == embeddings2.shape[0]) - assert(embeddings1.shape[1] == embeddings2.shape[1]) +def calculate_val(thresholds, + embeddings1, + embeddings2, + actual_issame, + far_target, + nrof_folds=10): + assert (embeddings1.shape[0] == embeddings2.shape[0]) + assert (embeddings1.shape[1] == embeddings2.shape[1]) nrof_pairs = min(len(actual_issame), embeddings1.shape[0]) nrof_thresholds = len(thresholds) k_fold = LFold(n_splits=nrof_folds, shuffle=False) - + val = np.zeros(nrof_folds) far = np.zeros(nrof_folds) - + diff = np.subtract(embeddings1, embeddings2) - dist = np.sum(np.square(diff),1) + dist = np.sum(np.square(diff), 1) indices = np.arange(nrof_pairs) - + for fold_idx, (train_set, test_set) in enumerate(k_fold.split(indices)): - + # Find the threshold that gives FAR = far_target far_train = np.zeros(nrof_thresholds) for threshold_idx, threshold in enumerate(thresholds): - _, far_train[threshold_idx] = calculate_val_far(threshold, dist[train_set], actual_issame[train_set]) - if np.max(far_train)>=far_target: + _, far_train[threshold_idx] = calculate_val_far( + threshold, dist[train_set], actual_issame[train_set]) + if np.max(far_train) >= far_target: f = interpolate.interp1d(far_train, thresholds, kind='slinear') threshold = f(far_target) else: threshold = 0.0 - - val[fold_idx], far[fold_idx] = calculate_val_far(threshold, dist[test_set], actual_issame[test_set]) - + + val[fold_idx], far[fold_idx] = calculate_val_far( + threshold, dist[test_set], actual_issame[test_set]) + val_mean = np.mean(val) far_mean = np.mean(far) val_std = np.std(val) @@ -158,7 +179,8 @@ def calculate_val(thresholds, embeddings1, embeddings2, actual_issame, far_targe def calculate_val_far(threshold, dist, actual_issame): predict_issame = np.less(dist, threshold) true_accept = np.sum(np.logical_and(predict_issame, actual_issame)) - false_accept = np.sum(np.logical_and(predict_issame, np.logical_not(actual_issame))) + false_accept = np.sum( + np.logical_and(predict_issame, np.logical_not(actual_issame))) n_same = np.sum(actual_issame) n_diff = np.sum(np.logical_not(actual_issame)) #print(true_accept, false_accept) @@ -167,429 +189,492 @@ def calculate_val_far(threshold, dist, actual_issame): far = float(false_accept) / float(n_diff) return val, far -def evaluate(embeddings, actual_issame, nrof_folds=10, pca = 0): + +def evaluate(embeddings, actual_issame, nrof_folds=10, pca=0): # Calculate evaluation metrics thresholds = np.arange(0, 4, 0.01) embeddings1 = embeddings[0::2] embeddings2 = embeddings[1::2] - tpr, fpr, accuracy = calculate_roc(thresholds, embeddings1, embeddings2, - np.asarray(actual_issame), nrof_folds=nrof_folds, pca = pca) + tpr, fpr, accuracy = calculate_roc(thresholds, + embeddings1, + embeddings2, + np.asarray(actual_issame), + nrof_folds=nrof_folds, + pca=pca) thresholds = np.arange(0, 4, 0.001) - val, val_std, far = calculate_val(thresholds, embeddings1, embeddings2, - np.asarray(actual_issame), 1e-3, nrof_folds=nrof_folds) + val, val_std, far = calculate_val(thresholds, + embeddings1, + embeddings2, + np.asarray(actual_issame), + 1e-3, + nrof_folds=nrof_folds) return tpr, fpr, accuracy, val, val_std, far + def load_bin(path, image_size): - try: - with open(path, 'rb') as f: - bins, issame_list = pickle.load(f) #py2 - except UnicodeDecodeError as e: - with open(path, 'rb') as f: - bins, issame_list = pickle.load(f, encoding='bytes') #py3 - data_list = [] - for flip in [0,1]: - data = nd.empty((len(issame_list)*2, 3, image_size[0], image_size[1])) - data_list.append(data) - for i in range(len(issame_list)*2): - _bin = bins[i] - img = mx.image.imdecode(_bin) - if img.shape[1]!=image_size[0]: - img = mx.image.resize_short(img, image_size[0]) - img = nd.transpose(img, axes=(2, 0, 1)) - for flip in [0,1]: - if flip==1: - img = mx.ndarray.flip(data=img, axis=2) - data_list[flip][i][:] = img - if i%1000==0: - print('loading bin', i) - print(data_list[0].shape) - return (data_list, issame_list) + try: + with open(path, 'rb') as f: + bins, issame_list = pickle.load(f) #py2 + except UnicodeDecodeError as e: + with open(path, 'rb') as f: + bins, issame_list = pickle.load(f, encoding='bytes') #py3 + data_list = [] + for flip in [0, 1]: + data = nd.empty( + (len(issame_list) * 2, 3, image_size[0], image_size[1])) + data_list.append(data) + for i in range(len(issame_list) * 2): + _bin = bins[i] + img = mx.image.imdecode(_bin) + if img.shape[1] != image_size[0]: + img = mx.image.resize_short(img, image_size[0]) + img = nd.transpose(img, axes=(2, 0, 1)) + for flip in [0, 1]: + if flip == 1: + img = mx.ndarray.flip(data=img, axis=2) + data_list[flip][i][:] = img + if i % 1000 == 0: + print('loading bin', i) + print(data_list[0].shape) + return (data_list, issame_list) -def test(data_set, mx_model, batch_size, nfolds=10, data_extra = None, label_shape = None): - print('testing verification..') - data_list = data_set[0] - issame_list = data_set[1] - model = mx_model - embeddings_list = [] - if data_extra is not None: - _data_extra = nd.array(data_extra) - time_consumed = 0.0 - if label_shape is None: - _label = nd.ones( (batch_size,) ) - else: - _label = nd.ones( label_shape ) - for i in range( len(data_list) ): - data = data_list[i] - embeddings = None - ba = 0 - while ba0.0: - imga = data[ida].asnumpy().transpose( (1,2,0) )[...,::-1] #to bgr - imgb = data[idb].asnumpy().transpose( (1,2,0) )[...,::-1] - #print(imga.shape, imgb.shape, violate, asame, _dist) - if asame: - pouts.append( (imga, imgb, _dist, best_threshold, ida) ) - else: - nouts.append( (imga, imgb, _dist, best_threshold, ida) ) - - tpr = np.mean(tprs,0) - fpr = np.mean(fprs,0) - acc = np.mean(accuracy) - pouts = sorted(pouts, key = lambda x: x[2], reverse=True) - nouts = sorted(nouts, key = lambda x: x[2], reverse=False) - print(len(pouts), len(nouts)) - print('acc', acc) - gap = 10 - image_shape = (112,224,3) - out_dir = "./badcases" - if not os.path.exists(out_dir): - os.makedirs(out_dir) - if len(nouts)>0: - threshold = nouts[0][3] - else: - threshold = pouts[-1][3] - - for item in [(pouts, 'positive(false_negative).png'), (nouts, 'negative(false_positive).png')]: - cols = 4 - rows = 8000 - outs = item[0] - if len(outs)==0: - continue - #if len(outs)==9: - # cols = 3 - # rows = 3 +def test_badcase(data_set, + mx_model, + batch_size, + name='', + data_extra=None, + label_shape=None): + print('testing verification badcase..') + data_list = data_set[0] + issame_list = data_set[1] + model = mx_model + embeddings_list = [] + if data_extra is not None: + _data_extra = nd.array(data_extra) + time_consumed = 0.0 + if label_shape is None: + _label = nd.ones((batch_size, )) + else: + _label = nd.ones(label_shape) + for i in range(len(data_list)): + data = data_list[i] + embeddings = None + ba = 0 + while ba < data.shape[0]: + bb = min(ba + batch_size, data.shape[0]) + count = bb - ba + _data = nd.slice_axis(data, axis=0, begin=bb - batch_size, end=bb) + #print(_data.shape, _label.shape) + time0 = datetime.datetime.now() + if data_extra is None: + db = mx.io.DataBatch(data=(_data, ), label=(_label, )) + else: + db = mx.io.DataBatch(data=(_data, _data_extra), + label=(_label, )) + model.forward(db, is_train=False) + net_out = model.get_outputs() + _embeddings = net_out[0].asnumpy() + time_now = datetime.datetime.now() + diff = time_now - time0 + time_consumed += diff.total_seconds() + if embeddings is None: + embeddings = np.zeros((data.shape[0], _embeddings.shape[1])) + embeddings[ba:bb, :] = _embeddings[(batch_size - count):, :] + ba = bb + embeddings_list.append(embeddings) + embeddings = embeddings_list[0] + embeddings_list[1] + embeddings = sklearn.preprocessing.normalize(embeddings) + thresholds = np.arange(0, 4, 0.01) + actual_issame = np.asarray(issame_list) + nrof_folds = 10 + embeddings1 = embeddings[0::2] + embeddings2 = embeddings[1::2] + assert (embeddings1.shape[0] == embeddings2.shape[0]) + assert (embeddings1.shape[1] == embeddings2.shape[1]) + nrof_pairs = min(len(actual_issame), embeddings1.shape[0]) + nrof_thresholds = len(thresholds) + k_fold = LFold(n_splits=nrof_folds, shuffle=False) - _rows = int(math.ceil(len(outs)/cols)) - rows = min(rows, _rows) - hack = {} + tprs = np.zeros((nrof_folds, nrof_thresholds)) + fprs = np.zeros((nrof_folds, nrof_thresholds)) + accuracy = np.zeros((nrof_folds)) + indices = np.arange(nrof_pairs) - if name.startswith('cfp') and item[1].startswith('pos'): - hack = {0:'manual/238_13.jpg.jpg', 6:'manual/088_14.jpg.jpg', 10:'manual/470_14.jpg.jpg', 25:'manual/238_13.jpg.jpg', 28:'manual/143_11.jpg.jpg'} + diff = np.subtract(embeddings1, embeddings2) + dist = np.sum(np.square(diff), 1) + data = data_list[0] - filename = item[1] - if len(name)>0: - filename = name+"_"+filename - filename = os.path.join(out_dir, filename) - img = np.zeros( (image_shape[0]*rows+20, image_shape[1]*cols+(cols-1)*gap, 3), dtype=np.uint8 ) - img[:,:,:] = 255 - text_color = (0,0,153) - text_color = (255,178,102) - text_color = (153,255,51) - for outi, out in enumerate(outs): - row = outi//cols - col = outi%cols - if row==rows: - break - imga = out[0].copy() - imgb = out[1].copy() - if outi in hack: - idx = out[4] - print('noise idx',idx) - aa = hack[outi] - imgb = cv2.imread(aa) - #if aa==1: - # imgb = cv2.transpose(imgb) - # imgb = cv2.flip(imgb, 1) - #elif aa==3: - # imgb = cv2.transpose(imgb) - # imgb = cv2.flip(imgb, 0) - #else: - # for ii in range(2): - # imgb = cv2.transpose(imgb) - # imgb = cv2.flip(imgb, 1) - dist = out[2] - _img = np.concatenate( (imga, imgb), axis=1 ) - k = "%.3f"%dist - #print(k) - font = cv2.FONT_HERSHEY_SIMPLEX - cv2.putText(_img,k,(80,image_shape[0]//2+7), font, 0.6, text_color, 2) - #_filename = filename+"_%d.png"%outi - #cv2.imwrite(_filename, _img) - img[row*image_shape[0]:(row+1)*image_shape[0], (col*image_shape[1]+gap*col):((col+1)*image_shape[1]+gap*col),:] = _img - #threshold = outs[0][3] - font = cv2.FONT_HERSHEY_SIMPLEX - k = "threshold: %.3f"%threshold - cv2.putText(img,k,(img.shape[1]//2-70,img.shape[0]-5), font, 0.6, text_color, 2) - cv2.imwrite(filename, img) + pouts = [] + nouts = [] + + for fold_idx, (train_set, test_set) in enumerate(k_fold.split(indices)): + + # Find the best threshold for the fold + acc_train = np.zeros((nrof_thresholds)) + #print(train_set) + #print(train_set.__class__) + for threshold_idx, threshold in enumerate(thresholds): + p2 = dist[train_set] + p3 = actual_issame[train_set] + _, _, acc_train[threshold_idx] = calculate_accuracy( + threshold, p2, p3) + best_threshold_index = np.argmax(acc_train) + for threshold_idx, threshold in enumerate(thresholds): + tprs[fold_idx, + threshold_idx], fprs[fold_idx, + threshold_idx], _ = calculate_accuracy( + threshold, dist[test_set], + actual_issame[test_set]) + _, _, accuracy[fold_idx] = calculate_accuracy( + thresholds[best_threshold_index], dist[test_set], + actual_issame[test_set]) + best_threshold = thresholds[best_threshold_index] + for iid in test_set: + ida = iid * 2 + idb = ida + 1 + asame = actual_issame[iid] + _dist = dist[iid] + violate = _dist - best_threshold + if not asame: + violate *= -1.0 + if violate > 0.0: + imga = data[ida].asnumpy().transpose( + (1, 2, 0))[..., ::-1] #to bgr + imgb = data[idb].asnumpy().transpose((1, 2, 0))[..., ::-1] + #print(imga.shape, imgb.shape, violate, asame, _dist) + if asame: + pouts.append((imga, imgb, _dist, best_threshold, ida)) + else: + nouts.append((imga, imgb, _dist, best_threshold, ida)) + + tpr = np.mean(tprs, 0) + fpr = np.mean(fprs, 0) + acc = np.mean(accuracy) + pouts = sorted(pouts, key=lambda x: x[2], reverse=True) + nouts = sorted(nouts, key=lambda x: x[2], reverse=False) + print(len(pouts), len(nouts)) + print('acc', acc) + gap = 10 + image_shape = (112, 224, 3) + out_dir = "./badcases" + if not os.path.exists(out_dir): + os.makedirs(out_dir) + if len(nouts) > 0: + threshold = nouts[0][3] + else: + threshold = pouts[-1][3] + + for item in [(pouts, 'positive(false_negative).png'), + (nouts, 'negative(false_positive).png')]: + cols = 4 + rows = 8000 + outs = item[0] + if len(outs) == 0: + continue + #if len(outs)==9: + # cols = 3 + # rows = 3 + + _rows = int(math.ceil(len(outs) / cols)) + rows = min(rows, _rows) + hack = {} + + if name.startswith('cfp') and item[1].startswith('pos'): + hack = { + 0: 'manual/238_13.jpg.jpg', + 6: 'manual/088_14.jpg.jpg', + 10: 'manual/470_14.jpg.jpg', + 25: 'manual/238_13.jpg.jpg', + 28: 'manual/143_11.jpg.jpg' + } + + filename = item[1] + if len(name) > 0: + filename = name + "_" + filename + filename = os.path.join(out_dir, filename) + img = np.zeros((image_shape[0] * rows + 20, image_shape[1] * cols + + (cols - 1) * gap, 3), + dtype=np.uint8) + img[:, :, :] = 255 + text_color = (0, 0, 153) + text_color = (255, 178, 102) + text_color = (153, 255, 51) + for outi, out in enumerate(outs): + row = outi // cols + col = outi % cols + if row == rows: + break + imga = out[0].copy() + imgb = out[1].copy() + if outi in hack: + idx = out[4] + print('noise idx', idx) + aa = hack[outi] + imgb = cv2.imread(aa) + #if aa==1: + # imgb = cv2.transpose(imgb) + # imgb = cv2.flip(imgb, 1) + #elif aa==3: + # imgb = cv2.transpose(imgb) + # imgb = cv2.flip(imgb, 0) + #else: + # for ii in range(2): + # imgb = cv2.transpose(imgb) + # imgb = cv2.flip(imgb, 1) + dist = out[2] + _img = np.concatenate((imga, imgb), axis=1) + k = "%.3f" % dist + #print(k) + font = cv2.FONT_HERSHEY_SIMPLEX + cv2.putText(_img, k, (80, image_shape[0] // 2 + 7), font, 0.6, + text_color, 2) + #_filename = filename+"_%d.png"%outi + #cv2.imwrite(_filename, _img) + img[row * image_shape[0]:(row + 1) * image_shape[0], + (col * image_shape[1] + + gap * col):((col + 1) * image_shape[1] + gap * col), :] = _img + #threshold = outs[0][3] + font = cv2.FONT_HERSHEY_SIMPLEX + k = "threshold: %.3f" % threshold + cv2.putText(img, k, (img.shape[1] // 2 - 70, img.shape[0] - 5), font, + 0.6, text_color, 2) + cv2.imwrite(filename, img) + + +def dumpR(data_set, + mx_model, + batch_size, + name='', + data_extra=None, + label_shape=None): + print('dump verification embedding..') + data_list = data_set[0] + issame_list = data_set[1] + model = mx_model + embeddings_list = [] + if data_extra is not None: + _data_extra = nd.array(data_extra) + time_consumed = 0.0 + if label_shape is None: + _label = nd.ones((batch_size, )) + else: + _label = nd.ones(label_shape) + for i in range(len(data_list)): + data = data_list[i] + embeddings = None + ba = 0 + while ba < data.shape[0]: + bb = min(ba + batch_size, data.shape[0]) + count = bb - ba + _data = nd.slice_axis(data, axis=0, begin=bb - batch_size, end=bb) + #print(_data.shape, _label.shape) + time0 = datetime.datetime.now() + if data_extra is None: + db = mx.io.DataBatch(data=(_data, ), label=(_label, )) + else: + db = mx.io.DataBatch(data=(_data, _data_extra), + label=(_label, )) + model.forward(db, is_train=False) + net_out = model.get_outputs() + _embeddings = net_out[0].asnumpy() + time_now = datetime.datetime.now() + diff = time_now - time0 + time_consumed += diff.total_seconds() + if embeddings is None: + embeddings = np.zeros((data.shape[0], _embeddings.shape[1])) + embeddings[ba:bb, :] = _embeddings[(batch_size - count):, :] + ba = bb + embeddings_list.append(embeddings) + embeddings = embeddings_list[0] + embeddings_list[1] + embeddings = sklearn.preprocessing.normalize(embeddings) + actual_issame = np.asarray(issame_list) + outname = os.path.join('temp.bin') + with open(outname, 'wb') as f: + pickle.dump((embeddings, issame_list), + f, + protocol=pickle.HIGHEST_PROTOCOL) -def dumpR(data_set, mx_model, batch_size, name='', data_extra = None, label_shape = None): - print('dump verification embedding..') - data_list = data_set[0] - issame_list = data_set[1] - model = mx_model - embeddings_list = [] - if data_extra is not None: - _data_extra = nd.array(data_extra) - time_consumed = 0.0 - if label_shape is None: - _label = nd.ones( (batch_size,) ) - else: - _label = nd.ones( label_shape ) - for i in range( len(data_list) ): - data = data_list[i] - embeddings = None - ba = 0 - while ba0: - _max = [int(x) for x in args.max.split(',')] - assert len(_max)==2 - if len(epochs)>_max[1]: - epochs = epochs[_max[0]:_max[1]] + parser = argparse.ArgumentParser(description='do verification') + # general + parser.add_argument('--data-dir', default='', help='') + parser.add_argument('--model', + default='../model/softmax,50', + help='path to load model.') + parser.add_argument('--target', + default='lfw,cfp_ff,cfp_fp,agedb_30', + help='test targets.') + parser.add_argument('--gpu', default=0, type=int, help='gpu id') + parser.add_argument('--batch-size', default=32, type=int, help='') + parser.add_argument('--max', default='', type=str, help='') + parser.add_argument('--mode', default=0, type=int, help='') + parser.add_argument('--nfolds', default=10, type=int, help='') + args = parser.parse_args() + #sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'common')) + #import face_image + #prop = face_image.load_property(args.data_dir) + #image_size = prop.image_size + image_size = [112, 112] + print('image_size', image_size) + ctx = mx.gpu(args.gpu) + nets = [] + vec = args.model.split(',') + prefix = args.model.split(',')[0] + epochs = [] + if len(vec) == 1: + pdir = os.path.dirname(prefix) + for fname in os.listdir(pdir): + if not fname.endswith('.params'): + continue + _file = os.path.join(pdir, fname) + if _file.startswith(prefix): + epoch = int(fname.split('.')[0].split('-')[1]) + epochs.append(epoch) + epochs = sorted(epochs, reverse=True) + if len(args.max) > 0: + _max = [int(x) for x in args.max.split(',')] + assert len(_max) == 2 + if len(epochs) > _max[1]: + epochs = epochs[_max[0]:_max[1]] - else: - epochs = [int(x) for x in vec[1].split('|')] - print('model number', len(epochs)) - time0 = datetime.datetime.now() - for epoch in epochs: - print('loading',prefix, epoch) - sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) - #arg_params, aux_params = ch_dev(arg_params, aux_params, ctx) - all_layers = sym.get_internals() - sym = all_layers['fc1_output'] - model = mx.mod.Module(symbol=sym, context=ctx, label_names = None) - #model.bind(data_shapes=[('data', (args.batch_size, 3, image_size[0], image_size[1]))], label_shapes=[('softmax_label', (args.batch_size,))]) - model.bind(data_shapes=[('data', (args.batch_size, 3, image_size[0], image_size[1]))]) - model.set_params(arg_params, aux_params) - nets.append(model) - time_now = datetime.datetime.now() - diff = time_now - time0 - print('model loading time', diff.total_seconds()) - - ver_list = [] - ver_name_list = [] - for name in args.target.split(','): - path = os.path.join(args.data_dir,name+".bin") - if os.path.exists(path): - print('loading.. ', name) - data_set = load_bin(path, image_size) - ver_list.append(data_set) - ver_name_list.append(name) - - if args.mode==0: - for i in range(len(ver_list)): - results = [] - for model in nets: - acc1, std1, acc2, std2, xnorm, embeddings_list = test(ver_list[i], model, args.batch_size, args.nfolds) - print('[%s]XNorm: %f' % (ver_name_list[i], xnorm)) - print('[%s]Accuracy: %1.5f+-%1.5f' % (ver_name_list[i], acc1, std1)) - print('[%s]Accuracy-Flip: %1.5f+-%1.5f' % (ver_name_list[i], acc2, std2)) - results.append(acc2) - print('Max of [%s] is %1.5f' % (ver_name_list[i], np.max(results))) - elif args.mode==1: - model = nets[0] - test_badcase(ver_list[0], model, args.batch_size, args.target) - else: - model = nets[0] - dumpR(ver_list[0], model, args.batch_size, args.target) + else: + epochs = [int(x) for x in vec[1].split('|')] + print('model number', len(epochs)) + time0 = datetime.datetime.now() + for epoch in epochs: + print('loading', prefix, epoch) + sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) + #arg_params, aux_params = ch_dev(arg_params, aux_params, ctx) + all_layers = sym.get_internals() + sym = all_layers['fc1_output'] + model = mx.mod.Module(symbol=sym, context=ctx, label_names=None) + #model.bind(data_shapes=[('data', (args.batch_size, 3, image_size[0], image_size[1]))], label_shapes=[('softmax_label', (args.batch_size,))]) + model.bind(data_shapes=[('data', (args.batch_size, 3, image_size[0], + image_size[1]))]) + model.set_params(arg_params, aux_params) + nets.append(model) + time_now = datetime.datetime.now() + diff = time_now - time0 + print('model loading time', diff.total_seconds()) + ver_list = [] + ver_name_list = [] + for name in args.target.split(','): + path = os.path.join(args.data_dir, name + ".bin") + if os.path.exists(path): + print('loading.. ', name) + data_set = load_bin(path, image_size) + ver_list.append(data_set) + ver_name_list.append(name) + if args.mode == 0: + for i in range(len(ver_list)): + results = [] + for model in nets: + acc1, std1, acc2, std2, xnorm, embeddings_list = test( + ver_list[i], model, args.batch_size, args.nfolds) + print('[%s]XNorm: %f' % (ver_name_list[i], xnorm)) + print('[%s]Accuracy: %1.5f+-%1.5f' % + (ver_name_list[i], acc1, std1)) + print('[%s]Accuracy-Flip: %1.5f+-%1.5f' % + (ver_name_list[i], acc2, std2)) + results.append(acc2) + print('Max of [%s] is %1.5f' % (ver_name_list[i], np.max(results))) + elif args.mode == 1: + model = nets[0] + test_badcase(ver_list[0], model, args.batch_size, args.target) + else: + model = nets[0] + dumpR(ver_list[0], model, args.batch_size, args.target) diff --git a/recognition/SubCenter-ArcFace/drop.py b/recognition/SubCenter-ArcFace/drop.py index 069af5d..24d20ba 100644 --- a/recognition/SubCenter-ArcFace/drop.py +++ b/recognition/SubCenter-ArcFace/drop.py @@ -17,186 +17,196 @@ import numpy as np sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'common')) from rec_builder import * + def get_embedding(args, imgrec, a, b, image_size, model): - ocontents = [] - for idx in range(a, b): - s = imgrec.read_idx(idx) - ocontents.append(s) - embeddings = None - #print(len(ocontents)) - ba = 0 - rlabel = -1 - imgs = [] - contents = [] - while True: - bb = min(ba+args.batch_size, len(ocontents)) - if ba>=bb: - break - _batch_size = bb-ba - #_batch_size2 = max(_batch_size, args.ctx_num) - _batch_size2 = _batch_size - if _batch_size%args.ctx_num!=0: - _batch_size2 = ((_batch_size//args.ctx_num)+1) * args.ctx_num - data = np.zeros( (_batch_size2,3, image_size[0], image_size[1]) ) - count = bb-ba - ii=0 - for i in range(ba, bb): - header, img = mx.recordio.unpack(ocontents[i]) - contents.append(img) - label = header.label - if not isinstance(label, numbers.Number): - label = label[0] - if rlabel<0: - rlabel = int(label) - - img = mx.image.imdecode(img) - rgb = img.asnumpy() - bgr = rgb[:,:,::-1] - imgs.append(bgr) - img = rgb.transpose( (2,0,1) ) - data[ii] = img - ii+=1 - while ii<_batch_size2: - data[ii] = data[0] - ii+=1 - nddata = nd.array(data) - db = mx.io.DataBatch(data=(nddata,)) - model.forward(db, is_train=False) - net_out = model.get_outputs() - net_out = net_out[0].asnumpy() - if embeddings is None: - embeddings = np.zeros( (len(ocontents), net_out.shape[1])) - embeddings[ba:bb,:] = net_out[0:_batch_size,:] - ba = bb - embeddings = sklearn.preprocessing.normalize(embeddings) - return embeddings, rlabel, contents + ocontents = [] + for idx in range(a, b): + s = imgrec.read_idx(idx) + ocontents.append(s) + embeddings = None + #print(len(ocontents)) + ba = 0 + rlabel = -1 + imgs = [] + contents = [] + while True: + bb = min(ba + args.batch_size, len(ocontents)) + if ba >= bb: + break + _batch_size = bb - ba + #_batch_size2 = max(_batch_size, args.ctx_num) + _batch_size2 = _batch_size + if _batch_size % args.ctx_num != 0: + _batch_size2 = ((_batch_size // args.ctx_num) + 1) * args.ctx_num + data = np.zeros((_batch_size2, 3, image_size[0], image_size[1])) + count = bb - ba + ii = 0 + for i in range(ba, bb): + header, img = mx.recordio.unpack(ocontents[i]) + contents.append(img) + label = header.label + if not isinstance(label, numbers.Number): + label = label[0] + if rlabel < 0: + rlabel = int(label) + + img = mx.image.imdecode(img) + rgb = img.asnumpy() + bgr = rgb[:, :, ::-1] + imgs.append(bgr) + img = rgb.transpose((2, 0, 1)) + data[ii] = img + ii += 1 + while ii < _batch_size2: + data[ii] = data[0] + ii += 1 + nddata = nd.array(data) + db = mx.io.DataBatch(data=(nddata, )) + model.forward(db, is_train=False) + net_out = model.get_outputs() + net_out = net_out[0].asnumpy() + if embeddings is None: + embeddings = np.zeros((len(ocontents), net_out.shape[1])) + embeddings[ba:bb, :] = net_out[0:_batch_size, :] + ba = bb + embeddings = sklearn.preprocessing.normalize(embeddings) + return embeddings, rlabel, contents + def main(args): - print(args) - image_size = (112,112) - print('image_size', image_size) - vec = args.model.split(',') - prefix = vec[0] - epoch = int(vec[1]) - print('loading',prefix, epoch) - ctx = [] - cvd = os.environ['CUDA_VISIBLE_DEVICES'].strip() - if len(cvd)>0: - for i in range(len(cvd.split(','))): - ctx.append(mx.gpu(i)) - if len(ctx)==0: - ctx = [mx.cpu()] - print('use cpu') - else: - print('gpu num:', len(ctx)) - args.ctx_num = len(ctx) - args.batch_size *= args.ctx_num - sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) - W = None - i = 0 - while True: - key = 'fc7_%d_weight'%i - i+=1 - if key not in arg_params: - break - _W = arg_params[key].asnumpy() - #_W = _W.reshape( (-1, 10, 512) ) - if W is None: - W = _W - else: - W = np.concatenate( (W, _W), axis=0 ) - K = args.k - W = sklearn.preprocessing.normalize(W) - W = W.reshape( (-1, K, 512) ) - all_layers = sym.get_internals() - sym = all_layers['fc1_output'] - model = mx.mod.Module(symbol=sym, context=ctx, label_names = None) - model.bind(data_shapes=[('data', (args.ctx_num, 3, image_size[0], image_size[1]))]) - model.set_params(arg_params, aux_params) - print('W:',W.shape) - path_imgrec = os.path.join(args.data, 'train.rec') - path_imgidx = os.path.join(args.data, 'train.idx') - imgrec = mx.recordio.MXIndexedRecordIO(path_imgidx, path_imgrec, 'r') # pylint: disable=redefined-variable-type - id_list = [] - s = imgrec.read_idx(0) - header, _ = mx.recordio.unpack(s) - assert header.flag>0 - print('header0 label', header.label) - header0 = (int(header.label[0]), int(header.label[1])) - #assert(header.flag==1) - imgidx = range(1, int(header.label[0])) - id2range = {} - a, b = int(header.label[0]), int(header.label[1]) - seq_identity = range(a,b) - print(len(seq_identity)) - image_count = 0 - pp=0 - for wid, identity in enumerate(seq_identity): - pp+=1 - s = imgrec.read_idx(identity) + print(args) + image_size = (112, 112) + print('image_size', image_size) + vec = args.model.split(',') + prefix = vec[0] + epoch = int(vec[1]) + print('loading', prefix, epoch) + ctx = [] + cvd = os.environ['CUDA_VISIBLE_DEVICES'].strip() + if len(cvd) > 0: + for i in range(len(cvd.split(','))): + ctx.append(mx.gpu(i)) + if len(ctx) == 0: + ctx = [mx.cpu()] + print('use cpu') + else: + print('gpu num:', len(ctx)) + args.ctx_num = len(ctx) + args.batch_size *= args.ctx_num + sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) + W = None + i = 0 + while True: + key = 'fc7_%d_weight' % i + i += 1 + if key not in arg_params: + break + _W = arg_params[key].asnumpy() + #_W = _W.reshape( (-1, 10, 512) ) + if W is None: + W = _W + else: + W = np.concatenate((W, _W), axis=0) + K = args.k + W = sklearn.preprocessing.normalize(W) + W = W.reshape((-1, K, 512)) + all_layers = sym.get_internals() + sym = all_layers['fc1_output'] + model = mx.mod.Module(symbol=sym, context=ctx, label_names=None) + model.bind(data_shapes=[('data', (args.ctx_num, 3, image_size[0], + image_size[1]))]) + model.set_params(arg_params, aux_params) + print('W:', W.shape) + path_imgrec = os.path.join(args.data, 'train.rec') + path_imgidx = os.path.join(args.data, 'train.idx') + imgrec = mx.recordio.MXIndexedRecordIO(path_imgidx, path_imgrec, 'r') # pylint: disable=redefined-variable-type + id_list = [] + s = imgrec.read_idx(0) header, _ = mx.recordio.unpack(s) - contents = [] - a,b = int(header.label[0]), int(header.label[1]) - _count = b-a - id_list.append( (wid, a, b, _count) ) - image_count += _count - pp = 0 - if not os.path.exists(args.output): - os.makedirs(args.output) - ret = np.zeros( (image_count, K+1), dtype=np.float32 ) - output_dir = args.output - builder = SeqRecBuilder(output_dir) - print(ret.shape) - imid = 0 - da = datetime.datetime.now() - label = 0 - num_images = 0 - cos_thresh = np.cos(np.pi*args.threshold / 180.0) - for id_item in id_list: - wid = id_item[0] - pp+=1 - if pp%40==0: - db = datetime.datetime.now() - print('processing id', pp, (db-da).total_seconds()) - da = db - x, _, contents = get_embedding(args, imgrec, id_item[1], id_item[2], image_size, model) - subcenters = W[wid] - K_stat = np.zeros( (K, ), dtype=np.int) - for i in range(x.shape[0]): - _x = x[i] - sim = np.dot(subcenters, _x) # len(sim)==K - mc = np.argmax(sim) - K_stat[mc] += 1 - dominant_index = np.argmax(K_stat) - dominant_center = subcenters[dominant_index] - sim = np.dot(x, dominant_center) - idx = np.where(sim>cos_thresh)[0] - num_drop = x.shape[0] - len(idx) - if len(idx)==0: - continue - #print("labelid %d dropped %d, from %d to %d"% (wid, num_drop, x.shape[0], len(idx))) - num_images += len(idx) - for _idx in idx: - c = contents[_idx] - builder.add(label, c, is_image=False) - label+=1 - builder.close() - - print('total:', num_images) - - + assert header.flag > 0 + print('header0 label', header.label) + header0 = (int(header.label[0]), int(header.label[1])) + #assert(header.flag==1) + imgidx = range(1, int(header.label[0])) + id2range = {} + a, b = int(header.label[0]), int(header.label[1]) + seq_identity = range(a, b) + print(len(seq_identity)) + image_count = 0 + pp = 0 + for wid, identity in enumerate(seq_identity): + pp += 1 + s = imgrec.read_idx(identity) + header, _ = mx.recordio.unpack(s) + contents = [] + a, b = int(header.label[0]), int(header.label[1]) + _count = b - a + id_list.append((wid, a, b, _count)) + image_count += _count + pp = 0 + if not os.path.exists(args.output): + os.makedirs(args.output) + ret = np.zeros((image_count, K + 1), dtype=np.float32) + output_dir = args.output + builder = SeqRecBuilder(output_dir) + print(ret.shape) + imid = 0 + da = datetime.datetime.now() + label = 0 + num_images = 0 + cos_thresh = np.cos(np.pi * args.threshold / 180.0) + for id_item in id_list: + wid = id_item[0] + pp += 1 + if pp % 40 == 0: + db = datetime.datetime.now() + print('processing id', pp, (db - da).total_seconds()) + da = db + x, _, contents = get_embedding(args, imgrec, id_item[1], id_item[2], + image_size, model) + subcenters = W[wid] + K_stat = np.zeros((K, ), dtype=np.int) + for i in range(x.shape[0]): + _x = x[i] + sim = np.dot(subcenters, _x) # len(sim)==K + mc = np.argmax(sim) + K_stat[mc] += 1 + dominant_index = np.argmax(K_stat) + dominant_center = subcenters[dominant_index] + sim = np.dot(x, dominant_center) + idx = np.where(sim > cos_thresh)[0] + num_drop = x.shape[0] - len(idx) + if len(idx) == 0: + continue + #print("labelid %d dropped %d, from %d to %d"% (wid, num_drop, x.shape[0], len(idx))) + num_images += len(idx) + for _idx in idx: + c = contents[_idx] + builder.add(label, c, is_image=False) + label += 1 + builder.close() + print('total:', num_images) if __name__ == '__main__': - parser = argparse.ArgumentParser(description='') - # general - parser.add_argument('--data', default='/bigdata/faces_ms1m_full', type=str, help='') - parser.add_argument('--output', default='/bigdata/ms1m_full_k3drop075', type=str, help='') - parser.add_argument('--model', default='../Evaluation/IJB/pretrained_models/r50-arcfacesc-msf-k3z/model,2', help='path to load model.') - parser.add_argument('--batch-size', default=16, type=int, help='') - parser.add_argument('--threshold', default=75, type=float, help='') - parser.add_argument('--k', default=3, type=int, help='') - args = parser.parse_args() - main(args) - + parser = argparse.ArgumentParser(description='') + # general + parser.add_argument('--data', + default='/bigdata/faces_ms1m_full', + type=str, + help='') + parser.add_argument('--output', + default='/bigdata/ms1m_full_k3drop075', + type=str, + help='') + parser.add_argument( + '--model', + default= + '../Evaluation/IJB/pretrained_models/r50-arcfacesc-msf-k3z/model,2', + help='path to load model.') + parser.add_argument('--batch-size', default=16, type=int, help='') + parser.add_argument('--threshold', default=75, type=float, help='') + parser.add_argument('--k', default=3, type=int, help='') + args = parser.parse_args() + main(args) diff --git a/recognition/SubCenter-ArcFace/image_iter.py b/recognition/SubCenter-ArcFace/image_iter.py index 851e2f9..2a07977 100644 --- a/recognition/SubCenter-ArcFace/image_iter.py +++ b/recognition/SubCenter-ArcFace/image_iter.py @@ -23,67 +23,74 @@ logger = logging.getLogger() class FaceImageIter(io.DataIter): - - def __init__(self, batch_size, data_shape, - path_imgrec = None, - shuffle=False, aug_list=None, mean = None, - rand_mirror = False, cutoff = 0, color_jittering = 0, - images_filter = 0, - data_name='data', label_name='softmax_label', **kwargs): + def __init__(self, + batch_size, + data_shape, + path_imgrec=None, + shuffle=False, + aug_list=None, + mean=None, + rand_mirror=False, + cutoff=0, + color_jittering=0, + images_filter=0, + data_name='data', + label_name='softmax_label', + **kwargs): super(FaceImageIter, self).__init__() assert path_imgrec if path_imgrec: - logging.info('loading recordio %s...', - path_imgrec) - path_imgidx = path_imgrec[0:-4]+".idx" - self.imgrec = recordio.MXIndexedRecordIO(path_imgidx, path_imgrec, 'r') # pylint: disable=redefined-variable-type + logging.info('loading recordio %s...', path_imgrec) + path_imgidx = path_imgrec[0:-4] + ".idx" + self.imgrec = recordio.MXIndexedRecordIO(path_imgidx, path_imgrec, + 'r') # pylint: disable=redefined-variable-type s = self.imgrec.read_idx(0) header, _ = recordio.unpack(s) - if header.flag>0: - print('header0 label', header.label) - self.header0 = (int(header.label[0]), int(header.label[1])) - #assert(header.flag==1) - self.imgidx = list(range(1, int(header.label[0]))) - #self.imgidx = [] - #self.id2range = {} - #self.seq_identity = range(int(header.label[0]), int(header.label[1])) - #for identity in self.seq_identity: - # s = self.imgrec.read_idx(identity) - # header, _ = recordio.unpack(s) - # a,b = int(header.label[0]), int(header.label[1]) - # count = b-a - # if count 0: + print('header0 label', header.label) + self.header0 = (int(header.label[0]), int(header.label[1])) + #assert(header.flag==1) + self.imgidx = list(range(1, int(header.label[0]))) + #self.imgidx = [] + #self.id2range = {} + #self.seq_identity = range(int(header.label[0]), int(header.label[1])) + #for identity in self.seq_identity: + # s = self.imgrec.read_idx(identity) + # header, _ = recordio.unpack(s) + # a,b = int(header.label[0]), int(header.label[1]) + # count = b-a + # if count= len(self.seq): - raise StopIteration - idx = self.seq[self.cur] - self.cur += 1 - if self.imgrec is not None: - s = self.imgrec.read_idx(idx) - header, img = recordio.unpack(s) - label = header.label - if not isinstance(label, numbers.Number): - label = label[0] - return label, img, None, None - else: - label, fname, bbox, landmark = self.imglist[idx] - return label, self.read_image(fname), bbox, landmark + while True: + if self.cur >= len(self.seq): + raise StopIteration + idx = self.seq[self.cur] + self.cur += 1 + if self.imgrec is not None: + s = self.imgrec.read_idx(idx) + header, img = recordio.unpack(s) + label = header.label + if not isinstance(label, numbers.Number): + label = label[0] + return label, img, None, None + else: + label, fname, bbox, landmark = self.imglist[idx] + return label, self.read_image(fname), bbox, landmark else: s = self.imgrec.read() if s is None: @@ -132,65 +138,64 @@ class FaceImageIter(io.DataIter): return header.label, img, None, None def brightness_aug(self, src, x): - alpha = 1.0 + random.uniform(-x, x) - src *= alpha - return src + alpha = 1.0 + random.uniform(-x, x) + src *= alpha + return src def contrast_aug(self, src, x): - alpha = 1.0 + random.uniform(-x, x) - coef = nd.array([[[0.299, 0.587, 0.114]]]) - gray = src * coef - gray = (3.0 * (1.0 - alpha) / gray.size) * nd.sum(gray) - src *= alpha - src += gray - return src + alpha = 1.0 + random.uniform(-x, x) + coef = nd.array([[[0.299, 0.587, 0.114]]]) + gray = src * coef + gray = (3.0 * (1.0 - alpha) / gray.size) * nd.sum(gray) + src *= alpha + src += gray + return src def saturation_aug(self, src, x): - alpha = 1.0 + random.uniform(-x, x) - coef = nd.array([[[0.299, 0.587, 0.114]]]) - gray = src * coef - gray = nd.sum(gray, axis=2, keepdims=True) - gray *= (1.0 - alpha) - src *= alpha - src += gray - return src + alpha = 1.0 + random.uniform(-x, x) + coef = nd.array([[[0.299, 0.587, 0.114]]]) + gray = src * coef + gray = nd.sum(gray, axis=2, keepdims=True) + gray *= (1.0 - alpha) + src *= alpha + src += gray + return src def color_aug(self, img, x): - #augs = [self.brightness_aug, self.contrast_aug, self.saturation_aug] - #random.shuffle(augs) - #for aug in augs: - # #print(img.shape) - # img = aug(img, x) - # #print(img.shape) - #return img - return self.CJA(img) + #augs = [self.brightness_aug, self.contrast_aug, self.saturation_aug] + #random.shuffle(augs) + #for aug in augs: + # #print(img.shape) + # img = aug(img, x) + # #print(img.shape) + #return img + return self.CJA(img) def mirror_aug(self, img): - _rd = random.randint(0,1) - if _rd==1: - for c in range(img.shape[2]): - img[:,:,c] = np.fliplr(img[:,:,c]) - return img + _rd = random.randint(0, 1) + if _rd == 1: + for c in range(img.shape[2]): + img[:, :, c] = np.fliplr(img[:, :, c]) + return img def compress_aug(self, img): - from PIL import Image - from io import BytesIO - buf = BytesIO() - img = Image.fromarray(img.asnumpy(), 'RGB') - q = random.randint(2, 20) - img.save(buf, format='JPEG', quality=q) - buf = buf.getvalue() - img = Image.open(BytesIO(buf)) - return nd.array(np.asarray(img, 'float32')) - + from PIL import Image + from io import BytesIO + buf = BytesIO() + img = Image.fromarray(img.asnumpy(), 'RGB') + q = random.randint(2, 20) + img.save(buf, format='JPEG', quality=q) + buf = buf.getvalue() + img = Image.open(BytesIO(buf)) + return nd.array(np.asarray(img, 'float32')) def next(self): if not self.is_init: - self.reset() - self.is_init = True + self.reset() + self.is_init = True """Returns the next batch of data.""" #print('in next', self.cur, self.labelcur) - self.nbatch+=1 + self.nbatch += 1 batch_size = self.batch_size i = 0 try: @@ -201,42 +206,42 @@ class FaceImageIter(io.DataIter): except Exception as e: logging.debug('Invalid decoding, skipping: %s', str(e)) continue - if _data.shape[0]!=self.data_shape[1]: - _data = mx.image.resize_short(_data, self.data_shape[1]) + if _data.shape[0] != self.data_shape[1]: + _data = mx.image.resize_short(_data, self.data_shape[1]) _data = _data.asnumpy().astype(np.float32) - if self.rand_mirror and np.random.rand()<0.5: - _data = _data[:,::-1,:] - if self.color_jittering>0: - if self.color_jittering>1: - _rd = random.randint(0,1) - if _rd==1: - _data = self.compress_aug(_data) - _data = self.color_aug(_data, 0.125) + if self.rand_mirror and np.random.rand() < 0.5: + _data = _data[:, ::-1, :] + if self.color_jittering > 0: + if self.color_jittering > 1: + _rd = random.randint(0, 1) + if _rd == 1: + _data = self.compress_aug(_data) + _data = self.color_aug(_data, 0.125) #if self.nd_mean is not None: # _data -= self.nd_mean # _data *= 0.0078125 - if self.cutoff>0: - _rd = random.randint(0,1) - if _rd==1: - #print('do cutoff aug', self.cutoff) - centerh = random.randint(0, _data.shape[0]-1) - centerw = random.randint(0, _data.shape[1]-1) - half = self.cutoff//2 - starth = max(0, centerh-half) - endh = min(_data.shape[0], centerh+half) - startw = max(0, centerw-half) - endw = min(_data.shape[1], centerw+half) - #print(starth, endh, startw, endw, _data.shape) - _data[starth:endh, startw:endw, :] = 128 + if self.cutoff > 0: + _rd = random.randint(0, 1) + if _rd == 1: + #print('do cutoff aug', self.cutoff) + centerh = random.randint(0, _data.shape[0] - 1) + centerw = random.randint(0, _data.shape[1] - 1) + half = self.cutoff // 2 + starth = max(0, centerh - half) + endh = min(_data.shape[0], centerh + half) + startw = max(0, centerw - half) + endw = min(_data.shape[1], centerw + half) + #print(starth, endh, startw, endw, _data.shape) + _data[starth:endh, startw:endw, :] = 128 #_data -= 127.5 #_data /= 128.0 #self.batch_data[i] = self.postprocess_data(_data) - _data = _data.transpose( (2,0,1) ) + _data = _data.transpose((2, 0, 1)) self.batch_data[i] = _data self.batch_label[i] = label i += 1 except StopIteration: - if i0 - self.provide_data = iter_list[0].provide_data - self.provide_label = iter_list[0].provide_label - self.iter_list = iter_list - self.cur_iter = None + def __init__(self, iter_list): + assert len(iter_list) > 0 + self.provide_data = iter_list[0].provide_data + self.provide_label = iter_list[0].provide_label + self.iter_list = iter_list + self.cur_iter = None - def reset(self): - self.cur_iter.reset() - - def next(self): - self.cur_iter = random.choice(self.iter_list) - while True: - try: - ret = self.cur_iter.next() - except StopIteration: + def reset(self): self.cur_iter.reset() - continue - return ret - + def next(self): + self.cur_iter = random.choice(self.iter_list) + while True: + try: + ret = self.cur_iter.next() + except StopIteration: + self.cur_iter.reset() + continue + return ret diff --git a/recognition/SubCenter-ArcFace/parall_module_local_v1.py b/recognition/SubCenter-ArcFace/parall_module_local_v1.py index 8a962fb..5f68610 100644 --- a/recognition/SubCenter-ArcFace/parall_module_local_v1.py +++ b/recognition/SubCenter-ArcFace/parall_module_local_v1.py @@ -1,4 +1,3 @@ - ''' @author: insightface ''' @@ -20,11 +19,17 @@ from mxnet import io import mxnet.ndarray as nd from config import config + class ParallModule(BaseModule): - def __init__(self, symbol, data_names, label_names, - logger=logging, context=ctx.cpu(), work_load_list=None, - asymbol = None, - args = None): + def __init__(self, + symbol, + data_names, + label_names, + logger=logging, + context=ctx.cpu(), + work_load_list=None, + asymbol=None, + args=None): super(ParallModule, self).__init__(logger=logger) self._symbol = symbol self._asymbol = asymbol @@ -48,25 +53,32 @@ class ParallModule(BaseModule): self._ctx_cpu = mx.cpu() self._ctx_single_gpu = self._context[-1] self._fixed_param_names = None - self._curr_module = Module(self._symbol, self._data_names, self._label_names, logger=self.logger, - context=self._context, work_load_list=self._work_load_list, - fixed_param_names=self._fixed_param_names) + self._curr_module = Module(self._symbol, + self._data_names, + self._label_names, + logger=self.logger, + context=self._context, + work_load_list=self._work_load_list, + fixed_param_names=self._fixed_param_names) self._arcface_modules = [] self._ctx_class_start = [] for i in range(len(self._context)): - args._ctxid = i - _module = Module(self._asymbol(args), self._data_names, self._label_names, logger=self.logger, - context=mx.gpu(i), work_load_list=self._work_load_list, - fixed_param_names=self._fixed_param_names) - self._arcface_modules.append(_module) - _c = args.local_class_start + i*args.ctx_num_classes - self._ctx_class_start.append(_c) + args._ctxid = i + _module = Module(self._asymbol(args), + self._data_names, + self._label_names, + logger=self.logger, + context=mx.gpu(i), + work_load_list=self._work_load_list, + fixed_param_names=self._fixed_param_names) + self._arcface_modules.append(_module) + _c = args.local_class_start + i * args.ctx_num_classes + self._ctx_class_start.append(_c) self._usekv = False if self._usekv: - self._distkv = mx.kvstore.create('dist_sync') - self._kvinit = {} - + self._distkv = mx.kvstore.create('dist_sync') + self._kvinit = {} def _reset_bind(self): self.binded = False @@ -108,54 +120,72 @@ class ParallModule(BaseModule): g = _g.copy() x = _x.copy() for _module in self._arcface_modules: - _g, _x = _module.get_params() - ag = _g.copy() - ax = _x.copy() - g.update(ag) - x.update(ax) + _g, _x = _module.get_params() + ag = _g.copy() + ax = _x.copy() + g.update(ag) + x.update(ax) return g, x - def set_params(self, arg_params, aux_params, allow_missing=False, force_init=True, + def set_params(self, + arg_params, + aux_params, + allow_missing=False, + force_init=True, allow_extra=False): - g = arg_params - x = aux_params - #ag = {} - #ax = {} - rk = [] - for k in g: - v = g[k] - if k.startswith('fc7'): - p1 = k.find('_') - p2 = k.rfind('_') - _ctxid = int(k[p1+1:p2]) - self._arcface_modules[_ctxid].set_params({k:v}, {}) - rk.append(k) - for k in rk: - del g[k] - self._curr_module.set_params(g, x) - #self._arcface_module.set_params(ag, ax) + g = arg_params + x = aux_params + #ag = {} + #ax = {} + rk = [] + for k in g: + v = g[k] + if k.startswith('fc7'): + p1 = k.find('_') + p2 = k.rfind('_') + _ctxid = int(k[p1 + 1:p2]) + self._arcface_modules[_ctxid].set_params({k: v}, {}) + rk.append(k) + for k in rk: + del g[k] + self._curr_module.set_params(g, x) + #self._arcface_module.set_params(ag, ax) - - def init_params(self, initializer=Uniform(0.01), arg_params=None, aux_params=None, - allow_missing=False, force_init=False, allow_extra=False): + def init_params(self, + initializer=Uniform(0.01), + arg_params=None, + aux_params=None, + allow_missing=False, + force_init=False, + allow_extra=False): if self.params_initialized and not force_init: return assert self.binded, 'call bind before initializing the parameters' #TODO init the same weights with all work nodes - self._curr_module.init_params(initializer=initializer, arg_params=None, - aux_params=None, allow_missing=allow_missing, - force_init=force_init, allow_extra=allow_extra) + self._curr_module.init_params(initializer=initializer, + arg_params=None, + aux_params=None, + allow_missing=allow_missing, + force_init=force_init, + allow_extra=allow_extra) for _module in self._arcface_modules: - #_initializer = initializer - _initializer = mx.init.Normal(0.01) - _module.init_params(initializer=_initializer, arg_params=None, - aux_params=None, allow_missing=allow_missing, - force_init=force_init, allow_extra=allow_extra) + #_initializer = initializer + _initializer = mx.init.Normal(0.01) + _module.init_params(initializer=_initializer, + arg_params=None, + aux_params=None, + allow_missing=allow_missing, + force_init=force_init, + allow_extra=allow_extra) self.params_initialized = True - - def bind(self, data_shapes, label_shapes=None, for_training=True, - inputs_need_grad=False, force_rebind=False, shared_module=None): + def bind(self, + data_shapes, + label_shapes=None, + for_training=True, + inputs_need_grad=False, + force_rebind=False, + shared_module=None): print('in_bind', self.params_initialized, data_shapes, label_shapes) if self.params_initialized: arg_params, aux_params = self.get_params() @@ -173,37 +203,54 @@ class ParallModule(BaseModule): self.for_training = for_training self.inputs_need_grad = inputs_need_grad self.binded = True - self._curr_module.bind(data_shapes, label_shapes, for_training, inputs_need_grad, - force_rebind=False, shared_module=None) + self._curr_module.bind(data_shapes, + label_shapes, + for_training, + inputs_need_grad, + force_rebind=False, + shared_module=None) _data_shape = data_shapes[0][1] print('_data_shape', _data_shape, label_shapes) for _module in self._arcface_modules: - _module.bind([('data', (_data_shape[0]*self._num_workers, self._emb_size))], [('softmax_label', (_data_shape[0]*self._num_workers,))], for_training, True, - force_rebind=False, shared_module=None) + _module.bind( + [('data', + (_data_shape[0] * self._num_workers, self._emb_size))], + [('softmax_label', (_data_shape[0] * self._num_workers, ))], + for_training, + True, + force_rebind=False, + shared_module=None) if self.params_initialized: self.set_params(arg_params, aux_params) - def init_optimizer(self, kvstore='local', optimizer='sgd', - optimizer_params=(('learning_rate', 0.01),), force_init=False): + def init_optimizer(self, + kvstore='local', + optimizer='sgd', + optimizer_params=(('learning_rate', 0.01), ), + force_init=False): assert self.binded and self.params_initialized if self.optimizer_initialized and not force_init: self.logger.warning('optimizer already initialized, ignoring.') return - self._curr_module.init_optimizer(kvstore, optimizer, optimizer_params, + self._curr_module.init_optimizer(kvstore, + optimizer, + optimizer_params, force_init=force_init) for _module in self._arcface_modules: - _module.init_optimizer(kvstore, optimizer, optimizer_params, - force_init=force_init) + _module.init_optimizer(kvstore, + optimizer, + optimizer_params, + force_init=force_init) self.optimizer_initialized = True def kv_push(self, key, value): - #if value.context!=mx.cpu(): - # value = value.as_in_context(mx.cpu()) - if not key in self._kvinit: - self._distkv.init(key, nd.zeros_like(value)) - self._kvinit[key] = 1 - self._distkv.push(key, value) + #if value.context!=mx.cpu(): + # value = value.as_in_context(mx.cpu()) + if not key in self._kvinit: + self._distkv.init(key, nd.zeros_like(value)) + self._kvinit[key] = 1 + self._distkv.push(key, value) #get fc1 and partial fc7 def forward(self, data_batch, is_train=None): @@ -211,43 +258,41 @@ class ParallModule(BaseModule): #print('{fc7_weight[0][0]}', self._iter, g['fc7_0_weight'].asnumpy()[0][0]) #print('{pre_fc1_weight[0][0]}', self._iter, g['pre_fc1_weight'].asnumpy()[0][0]) - assert self.binded and self.params_initialized self._curr_module.forward(data_batch, is_train=is_train) if is_train: - self._iter+=1 - fc1, label = self._curr_module.get_outputs(merge_multi_context=True) - global_fc1 = fc1 - self.global_label = label.as_in_context(self._ctx_cpu) + self._iter += 1 + fc1, label = self._curr_module.get_outputs( + merge_multi_context=True) + global_fc1 = fc1 + self.global_label = label.as_in_context(self._ctx_cpu) - - for i, _module in enumerate(self._arcface_modules): - _label = self.global_label - self._ctx_class_start[i] - db_global_fc1 = io.DataBatch([global_fc1], [_label]) - _module.forward(db_global_fc1) #fc7 with margin + for i, _module in enumerate(self._arcface_modules): + _label = self.global_label - self._ctx_class_start[i] + db_global_fc1 = io.DataBatch([global_fc1], [_label]) + _module.forward(db_global_fc1) #fc7 with margin #print('forward end') - def get_ndarray(self, context, name, shape): - key = "%s_%s"%(name, context) - #print(key) - if not key in self._nd_cache: - v = nd.zeros( shape=shape, ctx = context) - self._nd_cache[key] = v - else: - v = self._nd_cache[key] - return v + key = "%s_%s" % (name, context) + #print(key) + if not key in self._nd_cache: + v = nd.zeros(shape=shape, ctx=context) + self._nd_cache[key] = v + else: + v = self._nd_cache[key] + return v def get_ndarray2(self, context, name, arr): - key = "%s_%s"%(name, context) - #print(key) - if not key in self._nd_cache: - v = nd.zeros( shape=arr.shape, ctx = context) - self._nd_cache[key] = v - else: - v = self._nd_cache[key] - arr.copyto(v) - return v + key = "%s_%s" % (name, context) + #print(key) + if not key in self._nd_cache: + v = nd.zeros(shape=arr.shape, ctx=context) + self._nd_cache[key] = v + else: + v = self._nd_cache[key] + arr.copyto(v) + return v def backward(self, out_grads=None): #print('in backward') @@ -255,126 +300,145 @@ class ParallModule(BaseModule): #tmp_ctx = self._ctx_cpu tmp_ctx = self._ctx_single_gpu fc7_outs = [] - ctx_fc7_max = self.get_ndarray(tmp_ctx, 'ctx_fc7_max', (self._batch_size, len(self._context))) + ctx_fc7_max = self.get_ndarray(tmp_ctx, 'ctx_fc7_max', + (self._batch_size, len(self._context))) #local_fc7_max = nd.zeros( (self.global_label.shape[0],1), ctx=mx.cpu()) arcface_module_outputs = [] for i, _module in enumerate(self._arcface_modules): - #_fc7 = _module.get_outputs(merge_multi_context=True)[0] - out = _module.get_outputs(merge_multi_context=True) - #print(out[0].shape) - #print(out[1].shape) - arcface_module_outputs.append(out) - _fc7 = out[0] - fc7_outs.append(_fc7) - _fc7_max = nd.max(_fc7, axis=1).as_in_context(tmp_ctx) - ctx_fc7_max[:,i] = _fc7_max + #_fc7 = _module.get_outputs(merge_multi_context=True)[0] + out = _module.get_outputs(merge_multi_context=True) + #print(out[0].shape) + #print(out[1].shape) + arcface_module_outputs.append(out) + _fc7 = out[0] + fc7_outs.append(_fc7) + _fc7_max = nd.max(_fc7, axis=1).as_in_context(tmp_ctx) + ctx_fc7_max[:, i] = _fc7_max - local_fc7_max = self.get_ndarray(tmp_ctx, 'local_fc7_max', (self._batch_size, 1)) + local_fc7_max = self.get_ndarray(tmp_ctx, 'local_fc7_max', + (self._batch_size, 1)) nd.max(ctx_fc7_max, axis=1, keepdims=True, out=local_fc7_max) global_fc7_max = local_fc7_max #local_fc7_sum = None - local_fc7_sum = self.get_ndarray(tmp_ctx, 'local_fc7_sum', (self._batch_size,1)) - local_fc7_sum[:,:] = 0.0 + local_fc7_sum = self.get_ndarray(tmp_ctx, 'local_fc7_sum', + (self._batch_size, 1)) + local_fc7_sum[:, :] = 0.0 for i, _module in enumerate(self._arcface_modules): - _max = self.get_ndarray2(fc7_outs[i].context, 'fc7_max', global_fc7_max) - fc7_outs[i] = nd.broadcast_sub(fc7_outs[i], _max) - fc7_outs[i] = nd.exp(fc7_outs[i]) - _sum = nd.sum(fc7_outs[i], axis=1, keepdims=True).as_in_context(tmp_ctx) - local_fc7_sum += _sum + _max = self.get_ndarray2(fc7_outs[i].context, 'fc7_max', + global_fc7_max) + fc7_outs[i] = nd.broadcast_sub(fc7_outs[i], _max) + fc7_outs[i] = nd.exp(fc7_outs[i]) + _sum = nd.sum(fc7_outs[i], axis=1, + keepdims=True).as_in_context(tmp_ctx) + local_fc7_sum += _sum global_fc7_sum = local_fc7_sum - if self._iter%self._verbose==0: - #_ctx = self._context[-1] - _ctx = self._ctx_cpu - _probs = [] - for i, _module in enumerate(self._arcface_modules): - _prob = self.get_ndarray2(_ctx, '_fc7_prob_%d'%i, fc7_outs[i]) - _probs.append(_prob) - fc7_prob = self.get_ndarray(_ctx, 'test_fc7_prob', (self._batch_size, self._ctx_num_classes*len(self._context))) - nd.concat(*_probs, dim=1, out=fc7_prob) - fc7_pred = nd.argmax(fc7_prob, axis=1) - local_label = self.global_label - self._local_class_start - #local_label = self.get_ndarray2(_ctx, 'test_label', local_label) - _pred = nd.equal(fc7_pred, local_label) - print('{fc7_acc}', self._iter, nd.mean(_pred).asnumpy()[0]) - + if self._iter % self._verbose == 0: + #_ctx = self._context[-1] + _ctx = self._ctx_cpu + _probs = [] + for i, _module in enumerate(self._arcface_modules): + _prob = self.get_ndarray2(_ctx, '_fc7_prob_%d' % i, + fc7_outs[i]) + _probs.append(_prob) + fc7_prob = self.get_ndarray( + _ctx, 'test_fc7_prob', + (self._batch_size, self._ctx_num_classes * len(self._context))) + nd.concat(*_probs, dim=1, out=fc7_prob) + fc7_pred = nd.argmax(fc7_prob, axis=1) + local_label = self.global_label - self._local_class_start + #local_label = self.get_ndarray2(_ctx, 'test_label', local_label) + _pred = nd.equal(fc7_pred, local_label) + print('{fc7_acc}', self._iter, nd.mean(_pred).asnumpy()[0]) #local_fc1_grad = [] #fc1_grad_ctx = self._ctx_cpu fc1_grad_ctx = self._ctx_single_gpu - local_fc1_grad = self.get_ndarray(fc1_grad_ctx, 'local_fc1_grad', (self._batch_size,self._emb_size)) - local_fc1_grad[:,:] = 0.0 + local_fc1_grad = self.get_ndarray(fc1_grad_ctx, 'local_fc1_grad', + (self._batch_size, self._emb_size)) + local_fc1_grad[:, :] = 0.0 total_eloss = [] celoss_verbose = 1000 - if self._iter%celoss_verbose==0: - fc7_celoss = self.get_ndarray(tmp_ctx, 'test_fc7_celoss', (self._batch_size,)) - fc7_celoss[:] = 0.0 + if self._iter % celoss_verbose == 0: + fc7_celoss = self.get_ndarray(tmp_ctx, 'test_fc7_celoss', + (self._batch_size, )) + fc7_celoss[:] = 0.0 for i, _module in enumerate(self._arcface_modules): - _sum = self.get_ndarray2(fc7_outs[i].context, 'fc7_sum', global_fc7_sum) - fc7_outs[i] = nd.broadcast_div(fc7_outs[i], _sum) - a = i*self._ctx_num_classes - b = (i+1)*self._ctx_num_classes - _label = self.global_label - self._ctx_class_start[i] - _label = self.get_ndarray2(fc7_outs[i].context, 'label', _label) - onehot_label = self.get_ndarray(fc7_outs[i].context, 'label_onehot', (self._batch_size, self._ctx_num_classes)) - nd.one_hot(_label, depth=self._ctx_num_classes, on_value = 1.0, off_value = 0.0, out=onehot_label) - #print(fc7_outs[i].shape, onehot_label.shape) + _sum = self.get_ndarray2(fc7_outs[i].context, 'fc7_sum', + global_fc7_sum) + fc7_outs[i] = nd.broadcast_div(fc7_outs[i], _sum) + a = i * self._ctx_num_classes + b = (i + 1) * self._ctx_num_classes + _label = self.global_label - self._ctx_class_start[i] + _label = self.get_ndarray2(fc7_outs[i].context, 'label', _label) + onehot_label = self.get_ndarray( + fc7_outs[i].context, 'label_onehot', + (self._batch_size, self._ctx_num_classes)) + nd.one_hot(_label, + depth=self._ctx_num_classes, + on_value=1.0, + off_value=0.0, + out=onehot_label) + #print(fc7_outs[i].shape, onehot_label.shape) - if self._iter%celoss_verbose==0: - _ce_loss = fc7_outs[i] * onehot_label - _ce_loss = nd.sum(_ce_loss, axis=1) - fc7_celoss += _ce_loss.as_in_context(tmp_ctx) - fc7_outs[i] -= onehot_label + if self._iter % celoss_verbose == 0: + _ce_loss = fc7_outs[i] * onehot_label + _ce_loss = nd.sum(_ce_loss, axis=1) + fc7_celoss += _ce_loss.as_in_context(tmp_ctx) + fc7_outs[i] -= onehot_label - out = arcface_module_outputs[i] - out_grads = [fc7_outs[i]] - for j in range(1, len(out)): - eloss = out[j] - #print('eloss%d:'%j, eloss.shape) - #print(out_grads[0].shape) - #egrad_shape = (out_grads[0].shape[0], eloss.shape[0]) - egrad_shape = eloss.shape - egrad = self.get_ndarray(fc7_outs[i].context, 'egrad%d'%j, egrad_shape) - #egrad[:][:] = 1.0/egrad_shape[0] - egrad[:][:] = 1.0 - out_grads.append(egrad) - if self._iter%self._verbose==0: - total_eloss.append(np.mean(eloss.asnumpy())) + out = arcface_module_outputs[i] + out_grads = [fc7_outs[i]] + for j in range(1, len(out)): + eloss = out[j] + #print('eloss%d:'%j, eloss.shape) + #print(out_grads[0].shape) + #egrad_shape = (out_grads[0].shape[0], eloss.shape[0]) + egrad_shape = eloss.shape + egrad = self.get_ndarray(fc7_outs[i].context, 'egrad%d' % j, + egrad_shape) + #egrad[:][:] = 1.0/egrad_shape[0] + egrad[:][:] = 1.0 + out_grads.append(egrad) + if self._iter % self._verbose == 0: + total_eloss.append(np.mean(eloss.asnumpy())) - _module.backward(out_grads = out_grads) - #ctx_fc1_grad = _module.get_input_grads()[0].as_in_context(mx.cpu()) - ctx_fc1_grad = self.get_ndarray2(fc1_grad_ctx, 'ctx_fc1_grad_%d'%i, _module.get_input_grads()[0]) - local_fc1_grad += ctx_fc1_grad + _module.backward(out_grads=out_grads) + #ctx_fc1_grad = _module.get_input_grads()[0].as_in_context(mx.cpu()) + ctx_fc1_grad = self.get_ndarray2(fc1_grad_ctx, + 'ctx_fc1_grad_%d' % i, + _module.get_input_grads()[0]) + local_fc1_grad += ctx_fc1_grad - if self._iter%self._verbose==0 and len(total_eloss)>0: - print('{eloss}', self._iter, np.mean(total_eloss)) + if self._iter % self._verbose == 0 and len(total_eloss) > 0: + print('{eloss}', self._iter, np.mean(total_eloss)) #if self._iter%self._verbose==0: - if self._iter%celoss_verbose==0: - ce_loss = nd.log(fc7_celoss) * -1.0 - ce_loss = nd.mean(ce_loss) - print('CELOSS,%d,%f'% (self._iter, ce_loss.asnumpy())) + if self._iter % celoss_verbose == 0: + ce_loss = nd.log(fc7_celoss) * -1.0 + ce_loss = nd.mean(ce_loss) + print('CELOSS,%d,%f' % (self._iter, ce_loss.asnumpy())) global_fc1_grad = local_fc1_grad - self._curr_module.backward(out_grads = [global_fc1_grad]) - + self._curr_module.backward(out_grads=[global_fc1_grad]) def update(self): assert self.binded and self.params_initialized and self.optimizer_initialized self._curr_module.update() for i, _module in enumerate(self._arcface_modules): - _module.update() + _module.update() mx.nd.waitall() - def get_outputs(self, merge_multi_context=True): assert self.binded and self.params_initialized - return self._curr_module.get_outputs(merge_multi_context=merge_multi_context) + return self._curr_module.get_outputs( + merge_multi_context=merge_multi_context) #return self._arcface_module.get_outputs(merge_multi_context=merge_multi_context) def get_input_grads(self, merge_multi_context=True): assert self.binded and self.params_initialized and self.inputs_need_grad - return self._curr_module.get_input_grads(merge_multi_context=merge_multi_context) + return self._curr_module.get_input_grads( + merge_multi_context=merge_multi_context) def update_metric(self, eval_metric, labels): assert self.binded and self.params_initialized @@ -390,17 +454,31 @@ class ParallModule(BaseModule): def forward_backward(self, data_batch): """A convenient function that calls both ``forward`` and ``backward``.""" - self.forward(data_batch, is_train=True) # get fc1 and partial fc7 + self.forward(data_batch, is_train=True) # get fc1 and partial fc7 self.backward() - def fit(self, train_data, eval_data=None, eval_metric='acc', - epoch_end_callback=None, batch_end_callback=None, kvstore='local', - optimizer='sgd', optimizer_params=(('learning_rate', 0.01),), + def fit(self, + train_data, + eval_data=None, + eval_metric='acc', + epoch_end_callback=None, + batch_end_callback=None, + kvstore='local', + optimizer='sgd', + optimizer_params=(('learning_rate', 0.01), ), eval_end_callback=None, - eval_batch_end_callback=None, initializer=Uniform(0.01), - arg_params=None, aux_params=None, allow_missing=False, - force_rebind=False, force_init=False, begin_epoch=0, num_epoch=None, - validation_metric=None, monitor=None, sparse_row_id_fn=None): + eval_batch_end_callback=None, + initializer=Uniform(0.01), + arg_params=None, + aux_params=None, + allow_missing=False, + force_rebind=False, + force_init=False, + begin_epoch=0, + num_epoch=None, + validation_metric=None, + monitor=None, + sparse_row_id_fn=None): """Trains the module parameters. Checkout `Module Tutorial `_ to see @@ -482,13 +560,19 @@ class ParallModule(BaseModule): assert num_epoch is not None, 'please specify number of epochs' assert arg_params is None and aux_params is None - self.bind(data_shapes=train_data.provide_data, label_shapes=train_data.provide_label, - for_training=True, force_rebind=force_rebind) + self.bind(data_shapes=train_data.provide_data, + label_shapes=train_data.provide_label, + for_training=True, + force_rebind=force_rebind) if monitor is not None: self.install_monitor(monitor) - self.init_params(initializer=initializer, arg_params=arg_params, aux_params=aux_params, - allow_missing=allow_missing, force_init=force_init) - self.init_optimizer(kvstore=kvstore, optimizer=optimizer, + self.init_params(initializer=initializer, + arg_params=arg_params, + aux_params=aux_params, + allow_missing=allow_missing, + force_init=force_init) + self.init_optimizer(kvstore=kvstore, + optimizer=optimizer, optimizer_params=optimizer_params) if validation_metric is None: @@ -536,7 +620,8 @@ class ParallModule(BaseModule): try: # pre fetch next batch next_data_batch = next(data_iter) - self.prepare(next_data_batch, sparse_row_id_fn=sparse_row_id_fn) + self.prepare(next_data_batch, + sparse_row_id_fn=sparse_row_id_fn) except StopIteration: end_of_batch = True @@ -547,7 +632,8 @@ class ParallModule(BaseModule): # eval_name_vals = epoch_eval_metric.get_name_value() if batch_end_callback is not None: - batch_end_params = BatchEndParam(epoch=epoch, nbatch=nbatch, + batch_end_params = BatchEndParam(epoch=epoch, + nbatch=nbatch, eval_metric=None, locals=locals()) batch_end_callback(batch_end_params) @@ -559,7 +645,7 @@ class ParallModule(BaseModule): #for name, val in eval_name_vals: # self.logger.info('Epoch[%d] Train-%s=%f', epoch, name, val) toc = time.time() - self.logger.info('Epoch[%d] Time cost=%.3f', epoch, (toc-tic)) + self.logger.info('Epoch[%d] Time cost=%.3f', epoch, (toc - tic)) # sync aux params across devices arg_params, aux_params = self.get_params() @@ -567,4 +653,3 @@ class ParallModule(BaseModule): # end of 1 epoch, reset the data-iter for another epoch train_data.reset() - diff --git a/recognition/SubCenter-ArcFace/sample_config.py b/recognition/SubCenter-ArcFace/sample_config.py index 92e9af2..084a5ae 100644 --- a/recognition/SubCenter-ArcFace/sample_config.py +++ b/recognition/SubCenter-ArcFace/sample_config.py @@ -12,7 +12,7 @@ config.net_se = 0 config.net_act = 'prelu' config.net_unit = 3 config.net_input = 1 -config.net_blocks = [1,4,6,2] +config.net_blocks = [1, 4, 6, 2] config.net_output = 'E' config.net_multiplier = 1.0 config.val_targets = ['lfw', 'cfp_fp', 'agedb_30'] @@ -26,7 +26,7 @@ config.data_cutoff = False config.data_color = 0 config.data_images_filter = 0 config.count_flops = True -config.memonger = False #not work now +config.memonger = False #not work now config.loss_K = 3 @@ -74,7 +74,7 @@ network.y2 = edict() network.y2.net_name = 'fmobilefacenet' network.y2.emb_size = 256 network.y2.net_output = 'GDC' -network.y2.net_blocks = [2,8,16,4] +network.y2.net_blocks = [2, 8, 16, 4] network.m1 = edict() network.m1.net_name = 'fmobilenet' @@ -110,7 +110,7 @@ network.vargfacenet = edict() network.vargfacenet.net_name = 'vargfacenet' network.vargfacenet.net_multiplier = 1.25 network.vargfacenet.emb_size = 512 -network.vargfacenet.net_output='J' +network.vargfacenet.net_output = 'J' # dataset settings dataset = edict() @@ -119,14 +119,14 @@ dataset.emore = edict() dataset.emore.dataset = 'emore' dataset.emore.dataset_path = '../datasets/faces_emore' dataset.emore.num_classes = 85742 -dataset.emore.image_shape = (112,112,3) +dataset.emore.image_shape = (112, 112, 3) dataset.emore.val_targets = ['lfw', 'cfp_fp', 'agedb_30'] dataset.retina = edict() dataset.retina.dataset = 'retina' dataset.retina.dataset_path = '../datasets/ms1m-retinaface-t1' dataset.retina.num_classes = 93431 -dataset.retina.image_shape = (112,112,3) +dataset.retina.image_shape = (112, 112, 3) dataset.retina.val_targets = ['lfw', 'cfp_fp', 'agedb_30'] loss = edict() @@ -205,21 +205,20 @@ default.models_root = './models' def generate_config(_network, _dataset, _loss): for k, v in loss[_loss].items(): - config[k] = v - if k in default: - default[k] = v + config[k] = v + if k in default: + default[k] = v for k, v in network[_network].items(): - config[k] = v - if k in default: - default[k] = v + config[k] = v + if k in default: + default[k] = v for k, v in dataset[_dataset].items(): - config[k] = v - if k in default: - default[k] = v + config[k] = v + if k in default: + default[k] = v config.loss = _loss config.network = _network config.dataset = _dataset config.num_workers = 1 if 'DMLC_NUM_WORKER' in os.environ: - config.num_workers = int(os.environ['DMLC_NUM_WORKER']) - + config.num_workers = int(os.environ['DMLC_NUM_WORKER']) diff --git a/recognition/SubCenter-ArcFace/train_parall.py b/recognition/SubCenter-ArcFace/train_parall.py index 20827de..d444f6f 100644 --- a/recognition/SubCenter-ArcFace/train_parall.py +++ b/recognition/SubCenter-ArcFace/train_parall.py @@ -1,9 +1,7 @@ - ''' @author: insightface ''' - import os import sys import math @@ -27,97 +25,155 @@ import fmnasnet import fdensenet import vargfacenet - logger = logging.getLogger() logger.setLevel(logging.INFO) - args = None - def parse_args(): - parser = argparse.ArgumentParser(description='Train parall face network') - # general - parser.add_argument('--dataset', default=default.dataset, help='dataset config') - parser.add_argument('--network', default=default.network, help='network config') - parser.add_argument('--loss', default=default.loss, help='loss config') - args, rest = parser.parse_known_args() - generate_config(args.network, args.dataset, args.loss) - parser.add_argument('--models-root', default=default.models_root, help='root directory to save model.') - parser.add_argument('--pretrained', default=default.pretrained, help='pretrained model to load') - parser.add_argument('--pretrained-epoch', type=int, default=default.pretrained_epoch, help='pretrained epoch to load') - parser.add_argument('--ckpt', type=int, default=default.ckpt, help='checkpoint saving option. 0: discard saving. 1: save when necessary. 2: always save') - parser.add_argument('--verbose', type=int, default=default.verbose, help='do verification testing and model saving every verbose batches') - parser.add_argument('--lr', type=float, default=default.lr, help='start learning rate') - parser.add_argument('--lr-steps', type=str, default=default.lr_steps, help='steps of lr changing') - parser.add_argument('--wd', type=float, default=default.wd, help='weight decay') - parser.add_argument('--mom', type=float, default=default.mom, help='momentum') - parser.add_argument('--frequent', type=int, default=default.frequent, help='') - parser.add_argument('--per-batch-size', type=int, default=default.per_batch_size, help='batch size in each context') - parser.add_argument('--kvstore', type=str, default=default.kvstore, help='kvstore setting') - parser.add_argument('--worker-id', type=int, default=0, help='worker id for dist training, starts from 0') - parser.add_argument('--extra-model-name', type=str, default='', help='extra model name') - args = parser.parse_args() - return args + parser = argparse.ArgumentParser(description='Train parall face network') + # general + parser.add_argument('--dataset', + default=default.dataset, + help='dataset config') + parser.add_argument('--network', + default=default.network, + help='network config') + parser.add_argument('--loss', default=default.loss, help='loss config') + args, rest = parser.parse_known_args() + generate_config(args.network, args.dataset, args.loss) + parser.add_argument('--models-root', + default=default.models_root, + help='root directory to save model.') + parser.add_argument('--pretrained', + default=default.pretrained, + help='pretrained model to load') + parser.add_argument('--pretrained-epoch', + type=int, + default=default.pretrained_epoch, + help='pretrained epoch to load') + parser.add_argument( + '--ckpt', + type=int, + default=default.ckpt, + help= + 'checkpoint saving option. 0: discard saving. 1: save when necessary. 2: always save' + ) + parser.add_argument( + '--verbose', + type=int, + default=default.verbose, + help='do verification testing and model saving every verbose batches') + parser.add_argument('--lr', + type=float, + default=default.lr, + help='start learning rate') + parser.add_argument('--lr-steps', + type=str, + default=default.lr_steps, + help='steps of lr changing') + parser.add_argument('--wd', + type=float, + default=default.wd, + help='weight decay') + parser.add_argument('--mom', + type=float, + default=default.mom, + help='momentum') + parser.add_argument('--frequent', + type=int, + default=default.frequent, + help='') + parser.add_argument('--per-batch-size', + type=int, + default=default.per_batch_size, + help='batch size in each context') + parser.add_argument('--kvstore', + type=str, + default=default.kvstore, + help='kvstore setting') + parser.add_argument('--worker-id', + type=int, + default=0, + help='worker id for dist training, starts from 0') + parser.add_argument('--extra-model-name', + type=str, + default='', + help='extra model name') + args = parser.parse_args() + return args def get_symbol_embedding(): - embedding = eval(config.net_name).get_symbol() - all_label = mx.symbol.Variable('softmax_label') - #embedding = mx.symbol.BlockGrad(embedding) - all_label = mx.symbol.BlockGrad(all_label) - out_list = [embedding, all_label] - out = mx.symbol.Group(out_list) - return out + embedding = eval(config.net_name).get_symbol() + all_label = mx.symbol.Variable('softmax_label') + #embedding = mx.symbol.BlockGrad(embedding) + all_label = mx.symbol.BlockGrad(all_label) + out_list = [embedding, all_label] + out = mx.symbol.Group(out_list) + return out + def get_symbol_arcface(args): - embedding = mx.symbol.Variable('data') - all_label = mx.symbol.Variable('softmax_label') - gt_label = all_label - is_softmax = True - #print('call get_sym_arcface with', args, config) - if config.loss_name=='margin_softmax': - _weight = mx.symbol.Variable("fc7_%d_weight"%args._ctxid, shape=(args.ctx_num_classes*config.loss_K, config.emb_size), - lr_mult=config.fc7_lr_mult, wd_mult=config.fc7_wd_mult) - nweight = mx.symbol.L2Normalization(_weight, mode='instance') - nembedding = mx.symbol.L2Normalization(embedding, mode='instance', name='fc1n_%d'%args._ctxid) - fc7 = mx.sym.FullyConnected(data=nembedding, weight = nweight, no_bias = True, num_hidden=args.ctx_num_classes*config.loss_K, name='fc7_%d'%args._ctxid) - if config.loss_K>1: - sim_s3 = mx.symbol.reshape(fc7, (-1, args.ctx_num_classes, config.loss_K)) + embedding = mx.symbol.Variable('data') + all_label = mx.symbol.Variable('softmax_label') + gt_label = all_label + is_softmax = True + #print('call get_sym_arcface with', args, config) + if config.loss_name == 'margin_softmax': + _weight = mx.symbol.Variable("fc7_%d_weight" % args._ctxid, + shape=(args.ctx_num_classes * + config.loss_K, config.emb_size), + lr_mult=config.fc7_lr_mult, + wd_mult=config.fc7_wd_mult) + nweight = mx.symbol.L2Normalization(_weight, mode='instance') + nembedding = mx.symbol.L2Normalization(embedding, + mode='instance', + name='fc1n_%d' % args._ctxid) + fc7 = mx.sym.FullyConnected(data=nembedding, + weight=nweight, + no_bias=True, + num_hidden=args.ctx_num_classes * + config.loss_K, + name='fc7_%d' % args._ctxid) + if config.loss_K > 1: + sim_s3 = mx.symbol.reshape( + fc7, (-1, args.ctx_num_classes, config.loss_K)) - sim = mx.symbol.max(sim_s3, axis=2) - fc7 = sim + sim = mx.symbol.max(sim_s3, axis=2) + fc7 = sim - - - if config.loss_m1!=1.0 or config.loss_m2!=0.0 or config.loss_m3!=0.0: - gt_one_hot = mx.sym.one_hot(gt_label, depth = args.ctx_num_classes, on_value = 1.0, off_value = 0.0) - if config.loss_m1==1.0 and config.loss_m2==0.0: - _one_hot = gt_one_hot*args.margin_b - fc7 = fc7-_one_hot - else: - fc7_onehot = fc7 * gt_one_hot - cos_t = fc7_onehot - t = mx.sym.arccos(cos_t) - if config.loss_m1!=1.0: - t = t*config.loss_m1 - if config.loss_m2!=0.0: - t = t+config.loss_m2 - margin_cos = mx.sym.cos(t) - if config.loss_m3!=0.0: - margin_cos = margin_cos - config.loss_m3 - margin_fc7 = margin_cos - margin_fc7_onehot = margin_fc7 * gt_one_hot - diff = margin_fc7_onehot - fc7_onehot - fc7 = fc7+diff - fc7 = fc7*config.loss_s - out_list = [] - out_list.append(fc7) - if config.loss_name=='softmax': #softmax - out_list.append(gt_label) - out = mx.symbol.Group(out_list) - return out + if config.loss_m1 != 1.0 or config.loss_m2 != 0.0 or config.loss_m3 != 0.0: + gt_one_hot = mx.sym.one_hot(gt_label, + depth=args.ctx_num_classes, + on_value=1.0, + off_value=0.0) + if config.loss_m1 == 1.0 and config.loss_m2 == 0.0: + _one_hot = gt_one_hot * args.margin_b + fc7 = fc7 - _one_hot + else: + fc7_onehot = fc7 * gt_one_hot + cos_t = fc7_onehot + t = mx.sym.arccos(cos_t) + if config.loss_m1 != 1.0: + t = t * config.loss_m1 + if config.loss_m2 != 0.0: + t = t + config.loss_m2 + margin_cos = mx.sym.cos(t) + if config.loss_m3 != 0.0: + margin_cos = margin_cos - config.loss_m3 + margin_fc7 = margin_cos + margin_fc7_onehot = margin_fc7 * gt_one_hot + diff = margin_fc7_onehot - fc7_onehot + fc7 = fc7 + diff + fc7 = fc7 * config.loss_s + out_list = [] + out_list.append(fc7) + if config.loss_name == 'softmax': #softmax + out_list.append(gt_label) + out = mx.symbol.Group(out_list) + return out def train_net(args): @@ -127,26 +183,31 @@ def train_net(args): #mx.random.seed(_seed) ctx = [] cvd = os.environ['CUDA_VISIBLE_DEVICES'].strip() - if len(cvd)>0: - for i in range(len(cvd.split(','))): - ctx.append(mx.gpu(i)) - if len(ctx)==0: - ctx = [mx.cpu()] - print('use cpu') + if len(cvd) > 0: + for i in range(len(cvd.split(','))): + ctx.append(mx.gpu(i)) + if len(ctx) == 0: + ctx = [mx.cpu()] + print('use cpu') else: - print('gpu num:', len(ctx)) - if len(args.extra_model_name)==0: - prefix = os.path.join(args.models_root, '%s-%s-%s'%(args.network, args.loss, args.dataset), 'model') + print('gpu num:', len(ctx)) + if len(args.extra_model_name) == 0: + prefix = os.path.join( + args.models_root, + '%s-%s-%s' % (args.network, args.loss, args.dataset), 'model') else: - prefix = os.path.join(args.models_root, '%s-%s-%s-%s'%(args.network, args.loss, args.dataset, args.extra_model_name), 'model') + prefix = os.path.join( + args.models_root, '%s-%s-%s-%s' % + (args.network, args.loss, args.dataset, args.extra_model_name), + 'model') prefix_dir = os.path.dirname(prefix) print('prefix', prefix) if not os.path.exists(prefix_dir): - os.makedirs(prefix_dir) + os.makedirs(prefix_dir) args.ctx_num = len(ctx) - if args.per_batch_size==0: - args.per_batch_size = 128 - args.batch_size = args.per_batch_size*args.ctx_num + if args.per_batch_size == 0: + args.per_batch_size = 128 + args.batch_size = args.per_batch_size * args.ctx_num args.rescale_threshold = 0 args.image_channel = config.image_shape[2] config.batch_size = args.batch_size @@ -155,20 +216,20 @@ def train_net(args): path_imgrec = None path_imglist = None image_size = config.image_shape[0:2] - assert len(image_size)==2 - assert image_size[0]==image_size[1] + assert len(image_size) == 2 + assert image_size[0] == image_size[1] print('image_size', image_size) print('num_classes', config.num_classes) path_imgrec = os.path.join(data_dir, "train.rec") - data_shape = (args.image_channel,image_size[0],image_size[1]) + data_shape = (args.image_channel, image_size[0], image_size[1]) num_workers = config.num_workers global_num_ctx = num_workers * args.ctx_num - if config.num_classes%global_num_ctx==0: - args.ctx_num_classes = config.num_classes//global_num_ctx + if config.num_classes % global_num_ctx == 0: + args.ctx_num_classes = config.num_classes // global_num_ctx else: - args.ctx_num_classes = config.num_classes//global_num_ctx+1 + args.ctx_num_classes = config.num_classes // global_num_ctx + 1 args.local_num_classes = args.ctx_num_classes * args.ctx_num args.local_class_start = args.local_num_classes * args.worker_id @@ -190,71 +251,75 @@ def train_net(args): base_mom = args.mom arg_params = None aux_params = None - if len(args.pretrained)==0: - esym = get_symbol_embedding() - asym = get_symbol_arcface + if len(args.pretrained) == 0: + esym = get_symbol_embedding() + asym = get_symbol_arcface else: - assert False + assert False - if config.num_workers==1: - from parall_module_local_v1 import ParallModule + if config.num_workers == 1: + from parall_module_local_v1 import ParallModule else: - from parall_module_dist import ParallModule + from parall_module_dist import ParallModule model = ParallModule( - context = ctx, - symbol = esym, - data_names = ['data'], - label_names = ['softmax_label'], - asymbol = asym, - args = args, + context=ctx, + symbol=esym, + data_names=['data'], + label_names=['softmax_label'], + asymbol=asym, + args=args, ) val_dataiter = None train_dataiter = FaceImageIter( - batch_size = args.batch_size, - data_shape = data_shape, - path_imgrec = path_imgrec, - shuffle = True, - rand_mirror = config.data_rand_mirror, - mean = mean, - cutoff = config.data_cutoff, - color_jittering = config.data_color, - images_filter = config.data_images_filter, + batch_size=args.batch_size, + data_shape=data_shape, + path_imgrec=path_imgrec, + shuffle=True, + rand_mirror=config.data_rand_mirror, + mean=mean, + cutoff=config.data_cutoff, + color_jittering=config.data_color, + images_filter=config.data_images_filter, ) - - - if config.net_name=='fresnet' or config.net_name=='fmobilefacenet': - initializer = mx.init.Xavier(rnd_type='gaussian', factor_type="out", magnitude=2) #resnet style + if config.net_name == 'fresnet' or config.net_name == 'fmobilefacenet': + initializer = mx.init.Xavier(rnd_type='gaussian', + factor_type="out", + magnitude=2) #resnet style else: - initializer = mx.init.Xavier(rnd_type='uniform', factor_type="in", magnitude=2) + initializer = mx.init.Xavier(rnd_type='uniform', + factor_type="in", + magnitude=2) - _rescale = 1.0/args.batch_size - opt = optimizer.SGD(learning_rate=base_lr, momentum=base_mom, wd=base_wd, rescale_grad=_rescale) + _rescale = 1.0 / args.batch_size + opt = optimizer.SGD(learning_rate=base_lr, + momentum=base_mom, + wd=base_wd, + rescale_grad=_rescale) _cb = mx.callback.Speedometer(args.batch_size, args.frequent) - ver_list = [] ver_name_list = [] for name in config.val_targets: - path = os.path.join(data_dir,name+".bin") - if os.path.exists(path): - data_set = verification.load_bin(path, image_size) - ver_list.append(data_set) - ver_name_list.append(name) - print('ver', name) - + path = os.path.join(data_dir, name + ".bin") + if os.path.exists(path): + data_set = verification.load_bin(path, image_size) + ver_list.append(data_set) + ver_name_list.append(name) + print('ver', name) def ver_test(nbatch): - results = [] - for i in range(len(ver_list)): - acc1, std1, acc2, std2, xnorm, embeddings_list = verification.test(ver_list[i], model, args.batch_size, 10, None, None) - print('[%s][%d]XNorm: %f' % (ver_name_list[i], nbatch, xnorm)) - #print('[%s][%d]Accuracy: %1.5f+-%1.5f' % (ver_name_list[i], nbatch, acc1, std1)) - print('[%s][%d]Accuracy-Flip: %1.5f+-%1.5f' % (ver_name_list[i], nbatch, acc2, std2)) - results.append(acc2) - return results - + results = [] + for i in range(len(ver_list)): + acc1, std1, acc2, std2, xnorm, embeddings_list = verification.test( + ver_list[i], model, args.batch_size, 10, None, None) + print('[%s][%d]XNorm: %f' % (ver_name_list[i], nbatch, xnorm)) + #print('[%s][%d]Accuracy: %1.5f+-%1.5f' % (ver_name_list[i], nbatch, acc1, std1)) + print('[%s][%d]Accuracy-Flip: %1.5f+-%1.5f' % + (ver_name_list[i], nbatch, acc2, std2)) + results.append(acc2) + return results highest_acc = [0.0, 0.0] #lfw and target #for i in range(len(ver_list)): @@ -263,88 +328,91 @@ def train_net(args): save_step = [0] lr_steps = [int(x) for x in args.lr_steps.split(',')] print('lr_steps', lr_steps) + def _batch_callback(param): - #global global_step - global_step[0]+=1 - mbatch = global_step[0] - for step in lr_steps: - if mbatch==step: - opt.lr *= 0.1 - print('lr change to', opt.lr) - break + #global global_step + global_step[0] += 1 + mbatch = global_step[0] + for step in lr_steps: + if mbatch == step: + opt.lr *= 0.1 + print('lr change to', opt.lr) + break - _cb(param) - if mbatch%1000==0: - print('lr-batch-epoch:',opt.lr,param.nbatch,param.epoch) + _cb(param) + if mbatch % 1000 == 0: + print('lr-batch-epoch:', opt.lr, param.nbatch, param.epoch) - if mbatch>=0 and mbatch%args.verbose==0: - acc_list = ver_test(mbatch) - save_step[0]+=1 - msave = save_step[0] - do_save = False - is_highest = False - if len(acc_list)>0: - #lfw_score = acc_list[0] - #if lfw_score>highest_acc[0]: - # highest_acc[0] = lfw_score - # if lfw_score>=0.998: - # do_save = True - score = sum(acc_list) - if acc_list[-1]>=highest_acc[-1]: - if acc_list[-1]>highest_acc[-1]: - is_highest = True - else: - if score>=highest_acc[0]: - is_highest = True - highest_acc[0] = score - highest_acc[-1] = acc_list[-1] - #if lfw_score>=0.99: - # do_save = True - if is_highest: - do_save = True - if args.ckpt==0: - do_save = False - elif args.ckpt==2: - do_save = True - elif args.ckpt==3: - msave = 1 + if mbatch >= 0 and mbatch % args.verbose == 0: + acc_list = ver_test(mbatch) + save_step[0] += 1 + msave = save_step[0] + do_save = False + is_highest = False + if len(acc_list) > 0: + #lfw_score = acc_list[0] + #if lfw_score>highest_acc[0]: + # highest_acc[0] = lfw_score + # if lfw_score>=0.998: + # do_save = True + score = sum(acc_list) + if acc_list[-1] >= highest_acc[-1]: + if acc_list[-1] > highest_acc[-1]: + is_highest = True + else: + if score >= highest_acc[0]: + is_highest = True + highest_acc[0] = score + highest_acc[-1] = acc_list[-1] + #if lfw_score>=0.99: + # do_save = True + if is_highest: + do_save = True + if args.ckpt == 0: + do_save = False + elif args.ckpt == 2: + do_save = True + elif args.ckpt == 3: + msave = 1 - if do_save: - print('saving', msave) - if config.ckpt_embedding: - arg, aux = model.get_export_params() - else: - arg, aux = model.get_params() - all_layers = model.symbol.get_internals() - _sym = all_layers['fc1_output'] - mx.model.save_checkpoint(prefix, msave, _sym, arg, aux) - print('[%d]Accuracy-Highest: %1.5f'%(mbatch, highest_acc[-1])) - if config.max_steps>0 and mbatch>config.max_steps: - sys.exit(0) + if do_save: + print('saving', msave) + if config.ckpt_embedding: + arg, aux = model.get_export_params() + else: + arg, aux = model.get_params() + all_layers = model.symbol.get_internals() + _sym = all_layers['fc1_output'] + mx.model.save_checkpoint(prefix, msave, _sym, arg, aux) + print('[%d]Accuracy-Highest: %1.5f' % (mbatch, highest_acc[-1])) + if config.max_steps > 0 and mbatch > config.max_steps: + sys.exit(0) epoch_cb = None train_dataiter = mx.io.PrefetchingIter(train_dataiter) - model.fit(train_dataiter, - begin_epoch = begin_epoch, - num_epoch = 999999, - eval_data = val_dataiter, + model.fit( + train_dataiter, + begin_epoch=begin_epoch, + num_epoch=999999, + eval_data=val_dataiter, #eval_metric = eval_metrics, - kvstore = args.kvstore, - optimizer = opt, + kvstore=args.kvstore, + optimizer=opt, #optimizer_params = optimizer_params, - initializer = initializer, - arg_params = arg_params, - aux_params = aux_params, - allow_missing = True, - batch_end_callback = _batch_callback, - epoch_end_callback = epoch_cb ) + initializer=initializer, + arg_params=arg_params, + aux_params=aux_params, + allow_missing=True, + batch_end_callback=_batch_callback, + epoch_end_callback=epoch_cb) + def main(): global args args = parse_args() train_net(args) + if __name__ == '__main__': main() - diff --git a/recognition/common/build_eval_pack.py b/recognition/common/build_eval_pack.py index 49d0b74..23208ce 100644 --- a/recognition/common/build_eval_pack.py +++ b/recognition/common/build_eval_pack.py @@ -10,10 +10,12 @@ import numpy as np import sys import os sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'common')) -sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'RetinaFace')) +sys.path.append( + os.path.join(os.path.dirname(__file__), '..', '..', 'RetinaFace')) import face_align from retinaface import RetinaFace + def to_rgb(img): w, h = img.shape ret = np.empty((w, h, 3), dtype=np.uint8) @@ -21,33 +23,34 @@ def to_rgb(img): return ret -def IOU(Reframe,GTframe): - x1 = Reframe[0]; - y1 = Reframe[1]; - width1 = Reframe[2]-Reframe[0]; - height1 = Reframe[3]-Reframe[1]; +def IOU(Reframe, GTframe): + x1 = Reframe[0] + y1 = Reframe[1] + width1 = Reframe[2] - Reframe[0] + height1 = Reframe[3] - Reframe[1] - x2 = GTframe[0] - y2 = GTframe[1] - width2 = GTframe[2]-GTframe[0] - height2 = GTframe[3]-GTframe[1] + x2 = GTframe[0] + y2 = GTframe[1] + width2 = GTframe[2] - GTframe[0] + height2 = GTframe[3] - GTframe[1] - endx = max(x1+width1,x2+width2) - startx = min(x1,x2) - width = width1+width2-(endx-startx) + endx = max(x1 + width1, x2 + width2) + startx = min(x1, x2) + width = width1 + width2 - (endx - startx) - endy = max(y1+height1,y2+height2) - starty = min(y1,y2) - height = height1+height2-(endy-starty) + endy = max(y1 + height1, y2 + height2) + starty = min(y1, y2) + height = height1 + height2 - (endy - starty) + + if width <= 0 or height <= 0: + ratio = 0 + else: + Area = width * height + Area1 = width1 * height1 + Area2 = width2 * height2 + ratio = Area * 1. / (Area1 + Area2 - Area) + return ratio - if width <=0 or height <= 0: - ratio = 0 - else: - Area = width*height - Area1 = width1*height1 - Area2 = width2*height2 - ratio = Area*1./(Area1+Area2-Area) - return ratio parser = argparse.ArgumentParser(description='Package eval images') # general @@ -65,60 +68,69 @@ detector = RetinaFace(args.det_prefix, 0, gpu_id, network='net3') target_size = 400 max_size = 800 + def get_norm_crop(image_path): - im = cv2.imread(image_path) - im_shape = im.shape - im_size_min = np.min(im_shape[0:2]) - im_size_max = np.max(im_shape[0:2]) - im_scale = float(target_size) / float(im_size_min) - # prevent bigger axis from being more than max_size: - if np.round(im_scale * im_size_max) > max_size: - im_scale = float(max_size) / float(im_size_max) - bbox, landmark = detector.detect(im, threshold=0.5, scales=[im_scale]) - #print(im.shape, bbox.shape, landmark.shape) - if bbox.shape[0]==0: - bbox, landmark = detector.detect(im, threshold=0.05, scales=[im_scale*0.75, im_scale, im_scale*2.0]) - print('refine', im.shape, bbox.shape, landmark.shape) - nrof_faces = bbox.shape[0] - if nrof_faces>0: - det = bbox[:,0:4] - img_size = np.asarray(im.shape)[0:2] - bindex = 0 - if nrof_faces>1: - bounding_box_size = (det[:,2]-det[:,0])*(det[:,3]-det[:,1]) - img_center = img_size / 2 - offsets = np.vstack([ (det[:,0]+det[:,2])/2-img_center[1], (det[:,1]+det[:,3])/2-img_center[0] ]) - offset_dist_squared = np.sum(np.power(offsets,2.0),0) - bindex = np.argmax(bounding_box_size-offset_dist_squared*2.0) # some extra weight on the centering - #_bbox = bounding_boxes[bindex, 0:4] - _landmark = landmark[bindex] - warped = face_align.norm_crop(im, landmark = _landmark, image_size=args.image_size, mode=args.align_mode) - return warped - else: - return None + im = cv2.imread(image_path) + im_shape = im.shape + im_size_min = np.min(im_shape[0:2]) + im_size_max = np.max(im_shape[0:2]) + im_scale = float(target_size) / float(im_size_min) + # prevent bigger axis from being more than max_size: + if np.round(im_scale * im_size_max) > max_size: + im_scale = float(max_size) / float(im_size_max) + bbox, landmark = detector.detect(im, threshold=0.5, scales=[im_scale]) + #print(im.shape, bbox.shape, landmark.shape) + if bbox.shape[0] == 0: + bbox, landmark = detector.detect( + im, + threshold=0.05, + scales=[im_scale * 0.75, im_scale, im_scale * 2.0]) + print('refine', im.shape, bbox.shape, landmark.shape) + nrof_faces = bbox.shape[0] + if nrof_faces > 0: + det = bbox[:, 0:4] + img_size = np.asarray(im.shape)[0:2] + bindex = 0 + if nrof_faces > 1: + bounding_box_size = (det[:, 2] - det[:, 0]) * (det[:, 3] - + det[:, 1]) + img_center = img_size / 2 + offsets = np.vstack([(det[:, 0] + det[:, 2]) / 2 - img_center[1], + (det[:, 1] + det[:, 3]) / 2 - img_center[0]]) + offset_dist_squared = np.sum(np.power(offsets, 2.0), 0) + bindex = np.argmax(bounding_box_size - offset_dist_squared * + 2.0) # some extra weight on the centering + #_bbox = bounding_boxes[bindex, 0:4] + _landmark = landmark[bindex] + warped = face_align.norm_crop(im, + landmark=_landmark, + image_size=args.image_size, + mode=args.align_mode) + return warped + else: + return None bins = [] issame_list = [] pp = 0 for line in open(os.path.join(args.data_dir, 'pairs_label.txt'), 'r'): - pp+=1 - if pp%100==0: - print('processing', pp) - line = line.strip().split() - assert len(line)==3 - path1 = os.path.join(args.data_dir, line[0]) - path2 = os.path.join(args.data_dir, line[1]) - im1 = get_norm_crop(path1) - im2 = get_norm_crop(path2) - issame = True - if line[2]=='0': - issame = False - issame_list.append(issame) - for im in [im1, im2]: - _, s = cv2.imencode('.jpg', im) - bins.append(s) + pp += 1 + if pp % 100 == 0: + print('processing', pp) + line = line.strip().split() + assert len(line) == 3 + path1 = os.path.join(args.data_dir, line[0]) + path2 = os.path.join(args.data_dir, line[1]) + im1 = get_norm_crop(path1) + im2 = get_norm_crop(path2) + issame = True + if line[2] == '0': + issame = False + issame_list.append(issame) + for im in [im1, im2]: + _, s = cv2.imencode('.jpg', im) + bins.append(s) with open(args.output, 'wb') as f: - pickle.dump((bins, issame_list), f, protocol=pickle.HIGHEST_PROTOCOL) - + pickle.dump((bins, issame_list), f, protocol=pickle.HIGHEST_PROTOCOL) diff --git a/recognition/common/face_align.py b/recognition/common/face_align.py new file mode 100644 index 0000000..4f48a76 --- /dev/null +++ b/recognition/common/face_align.py @@ -0,0 +1,71 @@ +import cv2 +import numpy as np +from skimage import transform as trans + +src1 = np.array([[51.642, 50.115], [57.617, 49.990], [35.740, 69.007], + [51.157, 89.050], [57.025, 89.702]], + dtype=np.float32) +#<--left +src2 = np.array([[45.031, 50.118], [65.568, 50.872], [39.677, 68.111], + [45.177, 86.190], [64.246, 86.758]], + dtype=np.float32) + +#---frontal +src3 = np.array([[39.730, 51.138], [72.270, 51.138], [56.000, 68.493], + [42.463, 87.010], [69.537, 87.010]], + dtype=np.float32) + +#-->right +src4 = np.array([[46.845, 50.872], [67.382, 50.118], [72.737, 68.111], + [48.167, 86.758], [67.236, 86.190]], + dtype=np.float32) + +#-->right profile +src5 = np.array([[54.796, 49.990], [60.771, 50.115], [76.673, 69.007], + [55.388, 89.702], [61.257, 89.050]], + dtype=np.float32) + +src = np.array([src1, src2, src3, src4, src5]) +src_map = {112: src, 224: src * 2} + +arcface_src = np.array( + [[38.2946, 51.6963], [73.5318, 51.5014], [56.0252, 71.7366], + [41.5493, 92.3655], [70.7299, 92.2041]], + dtype=np.float32) + +arcface_src = np.expand_dims(arcface_src, axis=0) + +# In[66]: + + +# lmk is prediction; src is template +def estimate_norm(lmk, image_size=112, mode='arcface'): + assert lmk.shape == (5, 2) + tform = trans.SimilarityTransform() + lmk_tran = np.insert(lmk, 2, values=np.ones(5), axis=1) + min_M = [] + min_index = [] + min_error = float('inf') + if mode == 'arcface': + assert image_size == 112 + src = arcface_src + else: + src = src_map[image_size] + for i in np.arange(src.shape[0]): + tform.estimate(lmk, src[i]) + M = tform.params[0:2, :] + results = np.dot(M, lmk_tran.T) + results = results.T + error = np.sum(np.sqrt(np.sum((results - src[i])**2, axis=1))) + # print(error) + if error < min_error: + min_error = error + min_M = M + min_index = i + return min_M, min_index + + +def norm_crop(img, landmark, image_size=112, mode='arcface'): + M, pose_index = estimate_norm(landmark, image_size, mode) + warped = cv2.warpAffine(img, M, (image_size, image_size), borderValue=0.0) + return warped diff --git a/recognition/common/flops_counter.py b/recognition/common/flops_counter.py index 64469ec..8094241 100644 --- a/recognition/common/flops_counter.py +++ b/recognition/common/flops_counter.py @@ -1,4 +1,3 @@ - ''' @author: insightface ''' @@ -16,99 +15,106 @@ import mxnet as mx def is_no_bias(attr): - ret = False - if 'no_bias' in attr and (attr['no_bias']==True or attr['no_bias']=='True'): - ret = True - return ret + ret = False + if 'no_bias' in attr and (attr['no_bias'] == True + or attr['no_bias'] == 'True'): + ret = True + return ret + def count_fc_flops(input_filter, output_filter, attr): - #print(input_filter, output_filter ,attr) - ret = 2*input_filter*output_filter - if is_no_bias(attr): - ret -= output_filter - return int(ret) + #print(input_filter, output_filter ,attr) + ret = 2 * input_filter * output_filter + if is_no_bias(attr): + ret -= output_filter + return int(ret) def count_conv_flops(input_shape, output_shape, attr): - kernel = attr['kernel'][1:-1].split(',') - kernel = [int(x) for x in kernel] + kernel = attr['kernel'][1:-1].split(',') + kernel = [int(x) for x in kernel] - #print('kernel', kernel) - if is_no_bias(attr): - ret = (2*input_shape[1]*kernel[0]*kernel[1]-1)*output_shape[2]*output_shape[3]*output_shape[1] - else: - ret = 2*input_shape[1]*kernel[0]*kernel[1]*output_shape[2]*output_shape[3]*output_shape[1] - num_group = 1 - if 'num_group' in attr: - num_group = int(attr['num_group']) - ret /= num_group - return int(ret) + #print('kernel', kernel) + if is_no_bias(attr): + ret = (2 * input_shape[1] * kernel[0] * kernel[1] - + 1) * output_shape[2] * output_shape[3] * output_shape[1] + else: + ret = 2 * input_shape[1] * kernel[0] * kernel[1] * output_shape[ + 2] * output_shape[3] * output_shape[1] + num_group = 1 + if 'num_group' in attr: + num_group = int(attr['num_group']) + ret /= num_group + return int(ret) def count_flops(sym, **data_shapes): - all_layers = sym.get_internals() - #print(all_layers) - arg_shapes, out_shapes, aux_shapes = all_layers.infer_shape(**data_shapes) - out_shape_dict = dict(zip(all_layers.list_outputs(), out_shapes)) + all_layers = sym.get_internals() + #print(all_layers) + arg_shapes, out_shapes, aux_shapes = all_layers.infer_shape(**data_shapes) + out_shape_dict = dict(zip(all_layers.list_outputs(), out_shapes)) - nodes = json.loads(sym.tojson())['nodes'] - nodeid_shape = {} - for nodeid, node in enumerate(nodes): - name = node['name'] - layer_name = name+"_output" - if layer_name in out_shape_dict: - nodeid_shape[nodeid] = out_shape_dict[layer_name] - #print(nodeid_shape) - FLOPs = 0 - for nodeid, node in enumerate(nodes): - flops = 0 - if node['op']=='Convolution': - output_shape = nodeid_shape[nodeid] - name = node['name'] - attr = node['attrs'] - input_nodeid = node['inputs'][0][0] - input_shape = nodeid_shape[input_nodeid] - flops = count_conv_flops(input_shape, output_shape, attr) - elif node['op']=='FullyConnected': - attr = node['attrs'] - output_shape = nodeid_shape[nodeid] - input_nodeid = node['inputs'][0][0] - input_shape = nodeid_shape[input_nodeid] - output_filter = output_shape[1] - input_filter = input_shape[1]*input_shape[2]*input_shape[3] - #assert len(input_shape)==4 and input_shape[2]==1 and input_shape[3]==1 - flops = count_fc_flops(input_filter, output_filter, attr) - #print(node, flops) - FLOPs += flops + nodes = json.loads(sym.tojson())['nodes'] + nodeid_shape = {} + for nodeid, node in enumerate(nodes): + name = node['name'] + layer_name = name + "_output" + if layer_name in out_shape_dict: + nodeid_shape[nodeid] = out_shape_dict[layer_name] + #print(nodeid_shape) + FLOPs = 0 + for nodeid, node in enumerate(nodes): + flops = 0 + if node['op'] == 'Convolution': + output_shape = nodeid_shape[nodeid] + name = node['name'] + attr = node['attrs'] + input_nodeid = node['inputs'][0][0] + input_shape = nodeid_shape[input_nodeid] + flops = count_conv_flops(input_shape, output_shape, attr) + elif node['op'] == 'FullyConnected': + attr = node['attrs'] + output_shape = nodeid_shape[nodeid] + input_nodeid = node['inputs'][0][0] + input_shape = nodeid_shape[input_nodeid] + output_filter = output_shape[1] + input_filter = input_shape[1] * input_shape[2] * input_shape[3] + #assert len(input_shape)==4 and input_shape[2]==1 and input_shape[3]==1 + flops = count_fc_flops(input_filter, output_filter, attr) + #print(node, flops) + FLOPs += flops + + return FLOPs - return FLOPs def flops_str(FLOPs): - preset = [ (1e12, 'T'), (1e9, 'G'), (1e6, 'M'), (1e3, 'K') ] + preset = [(1e12, 'T'), (1e9, 'G'), (1e6, 'M'), (1e3, 'K')] + + for p in preset: + if FLOPs // p[0] > 0: + N = FLOPs / p[0] + ret = "%.1f%s" % (N, p[1]) + return ret + ret = "%.1f" % (FLOPs) + return ret - for p in preset: - if FLOPs//p[0]>0: - N = FLOPs/p[0] - ret = "%.1f%s"%(N, p[1]) - return ret - ret = "%.1f"%(FLOPs) - return ret if __name__ == '__main__': - parser = argparse.ArgumentParser(description='flops counter') - # general - #parser.add_argument('--model', default='../models2/y2-arcface-retinat1/model,1', help='path to load model.') - #parser.add_argument('--model', default='../models2/r100fc-arcface-retinaa/model,1', help='path to load model.') - parser.add_argument('--model', default='../models2/r50fc-arcface-emore/model,1', help='path to load model.') - args = parser.parse_args() - _vec = args.model.split(',') - assert len(_vec)==2 - prefix = _vec[0] - epoch = int(_vec[1]) - print('loading',prefix, epoch) - sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) - all_layers = sym.get_internals() - sym = all_layers['fc1_output'] - FLOPs = count_flops(sym, data=(1,3,112,112)) - print('FLOPs:', FLOPs) - + parser = argparse.ArgumentParser(description='flops counter') + # general + #parser.add_argument('--model', default='../models2/y2-arcface-retinat1/model,1', help='path to load model.') + #parser.add_argument('--model', default='../models2/r100fc-arcface-retinaa/model,1', help='path to load model.') + parser.add_argument('--model', + default='../models2/r50fc-arcface-emore/model,1', + help='path to load model.') + args = parser.parse_args() + _vec = args.model.split(',') + assert len(_vec) == 2 + prefix = _vec[0] + epoch = int(_vec[1]) + print('loading', prefix, epoch) + sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) + all_layers = sym.get_internals() + sym = all_layers['fc1_output'] + FLOPs = count_flops(sym, data=(1, 3, 112, 112)) + print('FLOPs:', FLOPs) diff --git a/recognition/common/rec2image.py b/recognition/common/rec2image.py index 535df24..21e5ec4 100644 --- a/recognition/common/rec2image.py +++ b/recognition/common/rec2image.py @@ -14,50 +14,47 @@ import numpy as np def main(args): - include_datasets = args.include.split(',') - rec_list = [] - for ds in include_datasets: - path_imgrec = os.path.join(ds, 'train.rec') - path_imgidx = os.path.join(ds, 'train.idx') - imgrec = mx.recordio.MXIndexedRecordIO(path_imgidx, path_imgrec, 'r') # pylint: disable=redefined-variable-type - rec_list.append(imgrec) - if not os.path.exists(args.output): - os.makedirs(args.output) - for ds_id in range(len(rec_list)): - id_list = [] - imgrec = rec_list[ds_id] - s = imgrec.read_idx(0) - header, _ = mx.recordio.unpack(s) - assert header.flag>0 - print('header0 label', header.label) - header0 = (int(header.label[0]), int(header.label[1])) - seq_identity = range(int(header.label[0]), int(header.label[1])) - pp=0 - for identity in seq_identity: - id_dir = os.path.join(args.output, "%d_%d"%(ds_id, identity)) - os.makedirs(id_dir) - pp+=1 - if pp%10==0: - print('processing id', pp) - s = imgrec.read_idx(identity) - header, _ = mx.recordio.unpack(s) - imgid = 0 - for _idx in range(int(header.label[0]), int(header.label[1])): - s = imgrec.read_idx(_idx) - _header, _img = mx.recordio.unpack(s) - _img = mx.image.imdecode(_img).asnumpy()[:,:,::-1] # to bgr - image_path = os.path.join(id_dir, "%d.jpg"%imgid) - cv2.imwrite(image_path, _img) - imgid+=1 - - + include_datasets = args.include.split(',') + rec_list = [] + for ds in include_datasets: + path_imgrec = os.path.join(ds, 'train.rec') + path_imgidx = os.path.join(ds, 'train.idx') + imgrec = mx.recordio.MXIndexedRecordIO(path_imgidx, path_imgrec, 'r') # pylint: disable=redefined-variable-type + rec_list.append(imgrec) + if not os.path.exists(args.output): + os.makedirs(args.output) + for ds_id in range(len(rec_list)): + id_list = [] + imgrec = rec_list[ds_id] + s = imgrec.read_idx(0) + header, _ = mx.recordio.unpack(s) + assert header.flag > 0 + print('header0 label', header.label) + header0 = (int(header.label[0]), int(header.label[1])) + seq_identity = range(int(header.label[0]), int(header.label[1])) + pp = 0 + for identity in seq_identity: + id_dir = os.path.join(args.output, "%d_%d" % (ds_id, identity)) + os.makedirs(id_dir) + pp += 1 + if pp % 10 == 0: + print('processing id', pp) + s = imgrec.read_idx(identity) + header, _ = mx.recordio.unpack(s) + imgid = 0 + for _idx in range(int(header.label[0]), int(header.label[1])): + s = imgrec.read_idx(_idx) + _header, _img = mx.recordio.unpack(s) + _img = mx.image.imdecode(_img).asnumpy()[:, :, ::-1] # to bgr + image_path = os.path.join(id_dir, "%d.jpg" % imgid) + cv2.imwrite(image_path, _img) + imgid += 1 if __name__ == '__main__': - parser = argparse.ArgumentParser(description='do dataset merge') - # general - parser.add_argument('--include', default='', type=str, help='') - parser.add_argument('--output', default='', type=str, help='') - args = parser.parse_args() - main(args) - + parser = argparse.ArgumentParser(description='do dataset merge') + # general + parser.add_argument('--include', default='', type=str, help='') + parser.add_argument('--output', default='', type=str, help='') + args = parser.parse_args() + main(args) diff --git a/recognition/common/rec_builder.py b/recognition/common/rec_builder.py index 6250876..1d51715 100644 --- a/recognition/common/rec_builder.py +++ b/recognition/common/rec_builder.py @@ -18,10 +18,12 @@ class SeqRecBuilder(): self.widx = 0 if not os.path.exists(path): os.makedirs(path) - self.writer = mx.recordio.MXIndexedRecordIO(os.path.join(path, 'train.idx'), os.path.join(path, 'train.rec'), 'w') + self.writer = mx.recordio.MXIndexedRecordIO( + os.path.join(path, 'train.idx'), os.path.join(path, 'train.rec'), + 'w') self.label_stat = [-1, -1] - def add(self, label, img, is_image = True): + def add(self, label, img, is_image=True): #img should be BGR #if self.sis: # assert label>=self.last_label @@ -33,16 +35,16 @@ class SeqRecBuilder(): else: s = mx.recordio.pack(header, img) self.writer.write_idx(idx, s) - if self.label_stat[0]<0: + if self.label_stat[0] < 0: self.label_stat = [label, label] else: self.label_stat[0] = min(self.label_stat[0], label) self.label_stat[1] = max(self.label_stat[1], label) - def close(self): with open(os.path.join(self.path, 'property'), 'w') as f: - f.write("%d,%d,%d\n"%(self.label_stat[1]+1, self.image_size[0], self.image_size[1])) + f.write("%d,%d,%d\n" % (self.label_stat[1] + 1, self.image_size[0], + self.image_size[1])) class RecBuilder(): @@ -53,28 +55,33 @@ class RecBuilder(): self.widx = 1 if not os.path.exists(path): os.makedirs(path) - self.writer = mx.recordio.MXIndexedRecordIO(os.path.join(path, 'train.idx'), os.path.join(path, 'train.rec'), 'w') + self.writer = mx.recordio.MXIndexedRecordIO( + os.path.join(path, 'train.idx'), os.path.join(path, 'train.rec'), + 'w') self.label_stat = [-1, -1] self.identities = [] def add(self, label, imgs): #img should be BGR - assert label>=0 - assert label>self.last_label - assert len(imgs)>0 + assert label >= 0 + assert label > self.last_label + assert len(imgs) > 0 idflag = [self.widx, -1] for img in imgs: idx = self.widx self.widx += 1 header = mx.recordio.IRHeader(0, label, idx, 0) if isinstance(img, np.ndarray): - s = mx.recordio.pack_img(header, img, quality=95, img_fmt='.jpg') + s = mx.recordio.pack_img(header, + img, + quality=95, + img_fmt='.jpg') else: s = mx.recordio.pack(header, img) self.writer.write_idx(idx, s) idflag[1] = self.widx self.identities.append(idflag) - if self.label_stat[0]<0: + if self.label_stat[0] < 0: self.label_stat = [label, label] else: self.label_stat[0] = min(self.label_stat[0], label) @@ -98,5 +105,5 @@ class RecBuilder(): print('label stat:', self.label_stat) with open(os.path.join(self.path, 'property'), 'w') as f: - f.write("%d,%d,%d\n"%(self.label_stat[1]+1, self.image_size[0], self.image_size[1])) - + f.write("%d,%d,%d\n" % (self.label_stat[1] + 1, self.image_size[0], + self.image_size[1])) diff --git a/recognition/common/verification.py b/recognition/common/verification.py index 8972e50..f46942a 100644 --- a/recognition/common/verification.py +++ b/recognition/common/verification.py @@ -2,19 +2,19 @@ """ # MIT License -# +# # Copyright (c) 2016 David Sandberg -# +# # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: -# +# # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. -# +# # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE @@ -45,110 +45,131 @@ from mxnet import ndarray as nd class LFold: - def __init__(self, n_splits = 2, shuffle = False): - self.n_splits = n_splits - if self.n_splits>1: - self.k_fold = KFold(n_splits = n_splits, shuffle = shuffle) + def __init__(self, n_splits=2, shuffle=False): + self.n_splits = n_splits + if self.n_splits > 1: + self.k_fold = KFold(n_splits=n_splits, shuffle=shuffle) - def split(self, indices): - if self.n_splits>1: - return self.k_fold.split(indices) - else: - return [(indices, indices)] + def split(self, indices): + if self.n_splits > 1: + return self.k_fold.split(indices) + else: + return [(indices, indices)] -def calculate_roc(thresholds, embeddings1, embeddings2, actual_issame, nrof_folds=10, pca = 0): - assert(embeddings1.shape[0] == embeddings2.shape[0]) - assert(embeddings1.shape[1] == embeddings2.shape[1]) +def calculate_roc(thresholds, + embeddings1, + embeddings2, + actual_issame, + nrof_folds=10, + pca=0): + assert (embeddings1.shape[0] == embeddings2.shape[0]) + assert (embeddings1.shape[1] == embeddings2.shape[1]) nrof_pairs = min(len(actual_issame), embeddings1.shape[0]) nrof_thresholds = len(thresholds) k_fold = LFold(n_splits=nrof_folds, shuffle=False) - - tprs = np.zeros((nrof_folds,nrof_thresholds)) - fprs = np.zeros((nrof_folds,nrof_thresholds)) + + tprs = np.zeros((nrof_folds, nrof_thresholds)) + fprs = np.zeros((nrof_folds, nrof_thresholds)) accuracy = np.zeros((nrof_folds)) indices = np.arange(nrof_pairs) #print('pca', pca) - - if pca==0: - diff = np.subtract(embeddings1, embeddings2) - dist = np.sum(np.square(diff),1) - + + if pca == 0: + diff = np.subtract(embeddings1, embeddings2) + dist = np.sum(np.square(diff), 1) + for fold_idx, (train_set, test_set) in enumerate(k_fold.split(indices)): #print('train_set', train_set) #print('test_set', test_set) - if pca>0: - print('doing pca on', fold_idx) - embed1_train = embeddings1[train_set] - embed2_train = embeddings2[train_set] - _embed_train = np.concatenate( (embed1_train, embed2_train), axis=0 ) - #print(_embed_train.shape) - pca_model = PCA(n_components=pca) - pca_model.fit(_embed_train) - embed1 = pca_model.transform(embeddings1) - embed2 = pca_model.transform(embeddings2) - embed1 = sklearn.preprocessing.normalize(embed1) - embed2 = sklearn.preprocessing.normalize(embed2) - #print(embed1.shape, embed2.shape) - diff = np.subtract(embed1, embed2) - dist = np.sum(np.square(diff),1) - + if pca > 0: + print('doing pca on', fold_idx) + embed1_train = embeddings1[train_set] + embed2_train = embeddings2[train_set] + _embed_train = np.concatenate((embed1_train, embed2_train), axis=0) + #print(_embed_train.shape) + pca_model = PCA(n_components=pca) + pca_model.fit(_embed_train) + embed1 = pca_model.transform(embeddings1) + embed2 = pca_model.transform(embeddings2) + embed1 = sklearn.preprocessing.normalize(embed1) + embed2 = sklearn.preprocessing.normalize(embed2) + #print(embed1.shape, embed2.shape) + diff = np.subtract(embed1, embed2) + dist = np.sum(np.square(diff), 1) + # Find the best threshold for the fold acc_train = np.zeros((nrof_thresholds)) for threshold_idx, threshold in enumerate(thresholds): - _, _, acc_train[threshold_idx] = calculate_accuracy(threshold, dist[train_set], actual_issame[train_set]) + _, _, acc_train[threshold_idx] = calculate_accuracy( + threshold, dist[train_set], actual_issame[train_set]) best_threshold_index = np.argmax(acc_train) #print('threshold', thresholds[best_threshold_index]) for threshold_idx, threshold in enumerate(thresholds): - tprs[fold_idx,threshold_idx], fprs[fold_idx,threshold_idx], _ = calculate_accuracy(threshold, dist[test_set], actual_issame[test_set]) - _, _, accuracy[fold_idx] = calculate_accuracy(thresholds[best_threshold_index], dist[test_set], actual_issame[test_set]) - - tpr = np.mean(tprs,0) - fpr = np.mean(fprs,0) + tprs[fold_idx, + threshold_idx], fprs[fold_idx, + threshold_idx], _ = calculate_accuracy( + threshold, dist[test_set], + actual_issame[test_set]) + _, _, accuracy[fold_idx] = calculate_accuracy( + thresholds[best_threshold_index], dist[test_set], + actual_issame[test_set]) + + tpr = np.mean(tprs, 0) + fpr = np.mean(fprs, 0) return tpr, fpr, accuracy + def calculate_accuracy(threshold, dist, actual_issame): predict_issame = np.less(dist, threshold) tp = np.sum(np.logical_and(predict_issame, actual_issame)) fp = np.sum(np.logical_and(predict_issame, np.logical_not(actual_issame))) - tn = np.sum(np.logical_and(np.logical_not(predict_issame), np.logical_not(actual_issame))) + tn = np.sum( + np.logical_and(np.logical_not(predict_issame), + np.logical_not(actual_issame))) fn = np.sum(np.logical_and(np.logical_not(predict_issame), actual_issame)) - - tpr = 0 if (tp+fn==0) else float(tp) / float(tp+fn) - fpr = 0 if (fp+tn==0) else float(fp) / float(fp+tn) - acc = float(tp+tn)/dist.size + + tpr = 0 if (tp + fn == 0) else float(tp) / float(tp + fn) + fpr = 0 if (fp + tn == 0) else float(fp) / float(fp + tn) + acc = float(tp + tn) / dist.size return tpr, fpr, acc - -def calculate_val(thresholds, embeddings1, embeddings2, actual_issame, far_target, nrof_folds=10): - assert(embeddings1.shape[0] == embeddings2.shape[0]) - assert(embeddings1.shape[1] == embeddings2.shape[1]) +def calculate_val(thresholds, + embeddings1, + embeddings2, + actual_issame, + far_target, + nrof_folds=10): + assert (embeddings1.shape[0] == embeddings2.shape[0]) + assert (embeddings1.shape[1] == embeddings2.shape[1]) nrof_pairs = min(len(actual_issame), embeddings1.shape[0]) nrof_thresholds = len(thresholds) k_fold = LFold(n_splits=nrof_folds, shuffle=False) - + val = np.zeros(nrof_folds) far = np.zeros(nrof_folds) - + diff = np.subtract(embeddings1, embeddings2) - dist = np.sum(np.square(diff),1) + dist = np.sum(np.square(diff), 1) indices = np.arange(nrof_pairs) - + for fold_idx, (train_set, test_set) in enumerate(k_fold.split(indices)): - + # Find the threshold that gives FAR = far_target far_train = np.zeros(nrof_thresholds) for threshold_idx, threshold in enumerate(thresholds): - _, far_train[threshold_idx] = calculate_val_far(threshold, dist[train_set], actual_issame[train_set]) - if np.max(far_train)>=far_target: + _, far_train[threshold_idx] = calculate_val_far( + threshold, dist[train_set], actual_issame[train_set]) + if np.max(far_train) >= far_target: f = interpolate.interp1d(far_train, thresholds, kind='slinear') threshold = f(far_target) else: threshold = 0.0 - - val[fold_idx], far[fold_idx] = calculate_val_far(threshold, dist[test_set], actual_issame[test_set]) - + + val[fold_idx], far[fold_idx] = calculate_val_far( + threshold, dist[test_set], actual_issame[test_set]) + val_mean = np.mean(val) far_mean = np.mean(far) val_std = np.std(val) @@ -158,7 +179,8 @@ def calculate_val(thresholds, embeddings1, embeddings2, actual_issame, far_targe def calculate_val_far(threshold, dist, actual_issame): predict_issame = np.less(dist, threshold) true_accept = np.sum(np.logical_and(predict_issame, actual_issame)) - false_accept = np.sum(np.logical_and(predict_issame, np.logical_not(actual_issame))) + false_accept = np.sum( + np.logical_and(predict_issame, np.logical_not(actual_issame))) n_same = np.sum(actual_issame) n_diff = np.sum(np.logical_not(actual_issame)) #print(true_accept, false_accept) @@ -167,208 +189,235 @@ def calculate_val_far(threshold, dist, actual_issame): far = float(false_accept) / float(n_diff) return val, far -def evaluate(embeddings, actual_issame, nrof_folds=10, pca = 0): + +def evaluate(embeddings, actual_issame, nrof_folds=10, pca=0): # Calculate evaluation metrics thresholds = np.arange(0, 4, 0.01) embeddings1 = embeddings[0::2] embeddings2 = embeddings[1::2] - tpr, fpr, accuracy = calculate_roc(thresholds, embeddings1, embeddings2, - np.asarray(actual_issame), nrof_folds=nrof_folds, pca = pca) + tpr, fpr, accuracy = calculate_roc(thresholds, + embeddings1, + embeddings2, + np.asarray(actual_issame), + nrof_folds=nrof_folds, + pca=pca) thresholds = np.arange(0, 4, 0.001) - val, val_std, far = calculate_val(thresholds, embeddings1, embeddings2, - np.asarray(actual_issame), 1e-3, nrof_folds=nrof_folds) + val, val_std, far = calculate_val(thresholds, + embeddings1, + embeddings2, + np.asarray(actual_issame), + 1e-3, + nrof_folds=nrof_folds) return tpr, fpr, accuracy, val, val_std, far + def load_bin(path, image_size): - try: - with open(path, 'rb') as f: - bins, issame_list = pickle.load(f) #py2 - except UnicodeDecodeError as e: - with open(path, 'rb') as f: - bins, issame_list = pickle.load(f, encoding='bytes') #py3 - data_list = [] - for flip in [0,1]: - data = nd.empty((len(issame_list)*2, 3, image_size[0], image_size[1])) - data_list.append(data) - for i in range(len(issame_list)*2): - _bin = bins[i] - img = mx.image.imdecode(_bin) - if img.shape[1]!=image_size[0]: - img = mx.image.resize_short(img, image_size[0]) - img = nd.transpose(img, axes=(2, 0, 1)) - for flip in [0,1]: - if flip==1: - img = mx.ndarray.flip(data=img, axis=2) - data_list[flip][i][:] = img - if i%1000==0: - print('loading bin', i) - print(data_list[0].shape) - return (data_list, issame_list) + try: + with open(path, 'rb') as f: + bins, issame_list = pickle.load(f) #py2 + except UnicodeDecodeError as e: + with open(path, 'rb') as f: + bins, issame_list = pickle.load(f, encoding='bytes') #py3 + data_list = [] + for flip in [0, 1]: + data = nd.empty( + (len(issame_list) * 2, 3, image_size[0], image_size[1])) + data_list.append(data) + for i in range(len(issame_list) * 2): + _bin = bins[i] + img = mx.image.imdecode(_bin) + if img.shape[1] != image_size[0]: + img = mx.image.resize_short(img, image_size[0]) + img = nd.transpose(img, axes=(2, 0, 1)) + for flip in [0, 1]: + if flip == 1: + img = mx.ndarray.flip(data=img, axis=2) + data_list[flip][i][:] = img + if i % 1000 == 0: + print('loading bin', i) + print(data_list[0].shape) + return (data_list, issame_list) -def test(data_set, mx_model, batch_size, nfolds=10, data_extra = None, label_shape = None): - print('testing verification..') - data_list = data_set[0] - issame_list = data_set[1] - model = mx_model - embeddings_list = [] - if data_extra is not None: - _data_extra = nd.array(data_extra) - time_consumed = 0.0 - if label_shape is None: - _label = nd.ones( (batch_size,) ) - else: - _label = nd.ones( label_shape ) - for i in range( len(data_list) ): - data = data_list[i] - embeddings = None - ba = 0 - while ba0: - _max = [int(x) for x in args.max.split(',')] - assert len(_max)==2 - if len(epochs)>_max[1]: - epochs = epochs[_max[0]:_max[1]] + parser = argparse.ArgumentParser(description='do verification') + # general + parser.add_argument('--data-dir', default='', help='') + parser.add_argument('--model', + default='../model/softmax,50', + help='path to load model.') + parser.add_argument('--target', + default='lfw,cfp_ff,cfp_fp,agedb_30', + help='test targets.') + parser.add_argument('--gpu', default=0, type=int, help='gpu id') + parser.add_argument('--batch-size', default=32, type=int, help='') + parser.add_argument('--max', default='', type=str, help='') + parser.add_argument('--mode', default=0, type=int, help='') + parser.add_argument('--nfolds', default=10, type=int, help='') + args = parser.parse_args() + #sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'common')) + #import face_image + #prop = face_image.load_property(args.data_dir) + #image_size = prop.image_size + image_size = [112, 112] + print('image_size', image_size) + ctx = mx.gpu(args.gpu) + nets = [] + vec = args.model.split(',') + prefix = args.model.split(',')[0] + epochs = [] + if len(vec) == 1: + pdir = os.path.dirname(prefix) + for fname in os.listdir(pdir): + if not fname.endswith('.params'): + continue + _file = os.path.join(pdir, fname) + if _file.startswith(prefix): + epoch = int(fname.split('.')[0].split('-')[1]) + epochs.append(epoch) + epochs = sorted(epochs, reverse=True) + if len(args.max) > 0: + _max = [int(x) for x in args.max.split(',')] + assert len(_max) == 2 + if len(epochs) > _max[1]: + epochs = epochs[_max[0]:_max[1]] - else: - epochs = [int(x) for x in vec[1].split('|')] - print('model number', len(epochs)) - time0 = datetime.datetime.now() - for epoch in epochs: - print('loading',prefix, epoch) - sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) - #arg_params, aux_params = ch_dev(arg_params, aux_params, ctx) - all_layers = sym.get_internals() - sym = all_layers['fc1_output'] - model = mx.mod.Module(symbol=sym, context=ctx, label_names = None) - #model.bind(data_shapes=[('data', (args.batch_size, 3, image_size[0], image_size[1]))], label_shapes=[('softmax_label', (args.batch_size,))]) - model.bind(data_shapes=[('data', (args.batch_size, 3, image_size[0], image_size[1]))]) - model.set_params(arg_params, aux_params) - nets.append(model) - time_now = datetime.datetime.now() - diff = time_now - time0 - print('model loading time', diff.total_seconds()) - - ver_list = [] - ver_name_list = [] - for name in args.target.split(','): - path = os.path.join(args.data_dir,name+".bin") - if os.path.exists(path): - print('loading.. ', name) - data_set = load_bin(path, image_size) - ver_list.append(data_set) - ver_name_list.append(name) - - if args.mode==0: - for i in range(len(ver_list)): - results = [] - for model in nets: - acc1, std1, acc2, std2, xnorm, embeddings_list = test(ver_list[i], model, args.batch_size, args.nfolds) - print('[%s]XNorm: %f' % (ver_name_list[i], xnorm)) - print('[%s]Accuracy: %1.5f+-%1.5f' % (ver_name_list[i], acc1, std1)) - print('[%s]Accuracy-Flip: %1.5f+-%1.5f' % (ver_name_list[i], acc2, std2)) - results.append(acc2) - print('Max of [%s] is %1.5f' % (ver_name_list[i], np.max(results))) - elif args.mode==1: - model = nets[0] - test_badcase(ver_list[0], model, args.batch_size, args.target) - else: - model = nets[0] - dumpR(ver_list[0], model, args.batch_size, args.target) + else: + epochs = [int(x) for x in vec[1].split('|')] + print('model number', len(epochs)) + time0 = datetime.datetime.now() + for epoch in epochs: + print('loading', prefix, epoch) + sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) + #arg_params, aux_params = ch_dev(arg_params, aux_params, ctx) + all_layers = sym.get_internals() + sym = all_layers['fc1_output'] + model = mx.mod.Module(symbol=sym, context=ctx, label_names=None) + #model.bind(data_shapes=[('data', (args.batch_size, 3, image_size[0], image_size[1]))], label_shapes=[('softmax_label', (args.batch_size,))]) + model.bind(data_shapes=[('data', (args.batch_size, 3, image_size[0], + image_size[1]))]) + model.set_params(arg_params, aux_params) + nets.append(model) + time_now = datetime.datetime.now() + diff = time_now - time0 + print('model loading time', diff.total_seconds()) + ver_list = [] + ver_name_list = [] + for name in args.target.split(','): + path = os.path.join(args.data_dir, name + ".bin") + if os.path.exists(path): + print('loading.. ', name) + data_set = load_bin(path, image_size) + ver_list.append(data_set) + ver_name_list.append(name) + if args.mode == 0: + for i in range(len(ver_list)): + results = [] + for model in nets: + acc1, std1, acc2, std2, xnorm, embeddings_list = test( + ver_list[i], model, args.batch_size, args.nfolds) + print('[%s]XNorm: %f' % (ver_name_list[i], xnorm)) + print('[%s]Accuracy: %1.5f+-%1.5f' % + (ver_name_list[i], acc1, std1)) + print('[%s]Accuracy-Flip: %1.5f+-%1.5f' % + (ver_name_list[i], acc2, std2)) + results.append(acc2) + print('Max of [%s] is %1.5f' % (ver_name_list[i], np.max(results))) + elif args.mode == 1: + model = nets[0] + test_badcase(ver_list[0], model, args.batch_size, args.target) + else: + model = nets[0] + dumpR(ver_list[0], model, args.batch_size, args.target) diff --git a/recognition/partial_fc/mxnet/callbacks.py b/recognition/partial_fc/mxnet/callbacks.py index 10659bd..f151d1f 100644 --- a/recognition/partial_fc/mxnet/callbacks.py +++ b/recognition/partial_fc/mxnet/callbacks.py @@ -44,7 +44,8 @@ class CallBackVertification(object): self.model = model self.ver_list = [] self.ver_name_list = [] - self.init_dataset(val_targets=config.val_targets, data_dir= os.path.dirname(config.rec), + self.init_dataset(val_targets=config.val_targets, + data_dir=os.path.dirname(config.rec), image_size=(config.image_size, config.image_size)) def ver_test(self, num_update): @@ -52,12 +53,15 @@ class CallBackVertification(object): for i in range(len(self.ver_list)): acc1, std1, acc2, std2, xnorm, embeddings_list = verification.test( self.ver_list[i], self.model, 10, 10, None, None) - logging.info('[%s][%d]XNorm: %f' % (self.ver_name_list[i], num_update, xnorm)) - logging.info('[%s][%d]Accuracy-Flip: %1.5f+-%1.5f' % (self.ver_name_list[i], num_update, acc2, std2)) + logging.info('[%s][%d]XNorm: %f' % + (self.ver_name_list[i], num_update, xnorm)) + logging.info('[%s][%d]Accuracy-Flip: %1.5f+-%1.5f' % + (self.ver_name_list[i], num_update, acc2, std2)) if acc2 > self.highest_acc_list[i]: self.highest_acc_list[i] = acc2 logging.info( - '[%s][%d]Accuracy-Highest: %1.5f' % (self.ver_name_list[i], num_update, self.highest_acc_list[i])) + '[%s][%d]Accuracy-Highest: %1.5f' % + (self.ver_name_list[i], num_update, self.highest_acc_list[i])) results.append(acc2) def init_dataset(self, val_targets, data_dir, image_size): @@ -98,7 +102,9 @@ class CallBackModelSave(object): def __call__(self, param): num_update = param.num_update - if num_update in [self.max_step - 10, ] or (num_update % 10000 == 0 and num_update > 0): + if num_update in [ + self.max_step - 10, + ] or (num_update % 10000 == 0 and num_update > 0): # params arg, aux = self.model.get_export_params() @@ -114,16 +120,16 @@ class CallBackModelSave(object): new_arg[key] = hvd.allreduce(tensor, average=True) if self.rank == 0: - mx.model.save_checkpoint( - prefix=self.prefix + "_average", - epoch=0, symbol=_sym, - arg_params=new_arg, - aux_params=new_aux) - mx.model.save_checkpoint( - prefix=self.prefix, - epoch=0, symbol=_sym, - arg_params=arg, - aux_params=aux) + mx.model.save_checkpoint(prefix=self.prefix + "_average", + epoch=0, + symbol=_sym, + arg_params=new_arg, + aux_params=new_aux) + mx.model.save_checkpoint(prefix=self.prefix, + epoch=0, + symbol=_sym, + arg_params=arg, + aux_params=aux) # training is over if num_update > self.max_step > 0: @@ -164,9 +170,10 @@ class CallBackLogging(object): self.loss_metric = MetricNdarray() t = time.localtime() - self.summary_writer = SummaryWriter( - logdir=os.path.join(self.prefix_dir, "log_tensorboard", - "%s_%s_%s" % (str(t.tm_mon), str(t.tm_mday), str(t.tm_hour))), verbose=False) + self.summary_writer = SummaryWriter(logdir=os.path.join( + self.prefix_dir, "log_tensorboard", + "%s_%s_%s" % (str(t.tm_mon), str(t.tm_mday), str(t.tm_hour))), + verbose=False) def __call__(self, param): """Callback to Show speed @@ -183,7 +190,8 @@ class CallBackLogging(object): if count % self.frequent == 0: nd.waitall() try: - speed = self.frequent * self.batch_size / (time.time() - self.tic) + speed = self.frequent * self.batch_size / (time.time() - + self.tic) speed_total = speed * self.size except ZeroDivisionError: speed = float('inf') @@ -192,13 +200,15 @@ class CallBackLogging(object): # summary loss loss_scalar = self.loss_metric.get() self.summary_writer.add_scalar(tag="loss", - value=loss_scalar, global_step=param.num_update) - loss_str_format = "[%d][%s]:%.2f " % (param.num_epoch, "loss", loss_scalar) + value=loss_scalar, + global_step=param.num_update) + loss_str_format = "[%d][%s]:%.2f " % (param.num_epoch, "loss", + loss_scalar) self.loss_metric.reset() # summary speed - self.summary_writer.add_scalar( - tag="speed", - value=speed, global_step=param.num_update) + self.summary_writer.add_scalar(tag="speed", + value=speed, + global_step=param.num_update) self.summary_writer.flush() if self.rank == 0: logging.info( diff --git a/recognition/partial_fc/mxnet/default.py b/recognition/partial_fc/mxnet/default.py index 3910ae9..fd692de 100644 --- a/recognition/partial_fc/mxnet/default.py +++ b/recognition/partial_fc/mxnet/default.py @@ -22,6 +22,7 @@ config.backbone_lr = 0.1 config.memory_bank_lr = config.backbone_lr config.sample_ratio = 1.0 + def generate_config(loss_name, dataset, network): # loss @@ -56,7 +57,9 @@ def generate_config(loss_name, dataset, network): elif dataset == 'glint360k_8GPU': config.lr_steps = '200000,400000,500000,550000' - config.val_targets = ['agedb_30', 'calfw', 'cfp_ff', 'cfp_fp', 'cplfw', 'lfw', 'vgg2_fp'] + config.val_targets = [ + 'agedb_30', 'calfw', 'cfp_ff', 'cfp_fp', 'cplfw', 'lfw', 'vgg2_fp' + ] config.rec = '/train_tmp/celeb_deepglint/train.rec' config.num_classes = 360232 config.batch_size = 64 diff --git a/recognition/partial_fc/mxnet/evaluation/align_ijb.py b/recognition/partial_fc/mxnet/evaluation/align_ijb.py index b0009c1..fb60abd 100644 --- a/recognition/partial_fc/mxnet/evaluation/align_ijb.py +++ b/recognition/partial_fc/mxnet/evaluation/align_ijb.py @@ -4,12 +4,9 @@ import cv2 import numpy as np from skimage import transform as trans -src = np.array([ - [30.2946, 51.6963], - [65.5318, 51.5014], - [48.0252, 71.7366], - [33.5493, 92.3655], - [62.7299, 92.2041]], dtype=np.float32) +src = np.array([[30.2946, 51.6963], [65.5318, 51.5014], [48.0252, 71.7366], + [33.5493, 92.3655], [62.7299, 92.2041]], + dtype=np.float32) src[:, 0] += 8.0 img_path = '/data/anxiang/datasets/IJB_release/IJBC/loose_crop' @@ -25,7 +22,8 @@ for img_index, each_line in enumerate(files): name_lmk_score = each_line.strip().split(' ') img_name = os.path.join(img_path, name_lmk_score[0]) img = cv2.imread(img_name) - landmark = np.array([float(x) for x in name_lmk_score[1:-1]], dtype=np.float32) + landmark = np.array([float(x) for x in name_lmk_score[1:-1]], + dtype=np.float32) landmark = landmark.reshape((5, 2)) if landmark.shape[0] == 68: diff --git a/recognition/partial_fc/mxnet/evaluation/ijb.py b/recognition/partial_fc/mxnet/evaluation/ijb.py index 64a560d..4651f65 100644 --- a/recognition/partial_fc/mxnet/evaluation/ijb.py +++ b/recognition/partial_fc/mxnet/evaluation/ijb.py @@ -32,31 +32,32 @@ parser.add_argument('--result-dir', default='.', type=str, help='') parser.add_argument('--gpu', default='0', type=str, help='gpu id') parser.add_argument('--batch-size', default=128, type=int, help='') parser.add_argument('--job', default='insightface', type=str, help='job name') -parser.add_argument('--target', default='IJBC', type=str, help='target, set to IJBC or IJBB') +parser.add_argument('--target', + default='IJBC', + type=str, + help='target, set to IJBC or IJBB') args = parser.parse_args() os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu -target = args.target -model_path = args.model_prefix -image_path = args.image_path -result_dir = args.result_dir -epoch = args.model_epoch -use_norm_score = True # if Ture, TestMode(N1) -use_detector_score = True # if Ture, TestMode(D1) -use_flip_test = True # if Ture, TestMode(F1) -job = args.job -batch_size = args.batch_size +target = args.target +model_path = args.model_prefix +image_path = args.image_path +result_dir = args.result_dir +epoch = args.model_epoch +use_norm_score = True # if Ture, TestMode(N1) +use_detector_score = True # if Ture, TestMode(D1) +use_flip_test = True # if Ture, TestMode(F1) +job = args.job +batch_size = args.batch_size class DatasetIJB(Dataset): def __init__(self, root, lines, align=True): - self.src = np.array([ - [30.2946, 51.6963], - [65.5318, 51.5014], - [48.0252, 71.7366], - [33.5493, 92.3655], - [62.7299, 92.2041]], dtype=np.float32) + self.src = np.array( + [[30.2946, 51.6963], [65.5318, 51.5014], [48.0252, 71.7366], + [33.5493, 92.3655], [62.7299, 92.2041]], + dtype=np.float32) self.src[:, 0] += 8.0 self.lines = lines self.img_root = root @@ -72,7 +73,8 @@ class DatasetIJB(Dataset): img = cv2.imread(img_name) if self.align: - landmark = np.array([float(x) for x in name_lmk_score[1:-1]], dtype=np.float32) + landmark = np.array([float(x) for x in name_lmk_score[1:-1]], + dtype=np.float32) landmark = landmark.reshape((5, 2)) # assert landmark.shape[0] == 68 or landmark.shape[0] == 5 @@ -112,28 +114,35 @@ def extract_parallel(prefix, epoch, dataset, batch_size, size): def batchify_fn(data): return mx.nd.concat(*data, dim=0) - data_loader = DataLoader(dataset, batch_size, last_batch='keep', num_workers=8, thread_pool=True, - prefetch=64, batchify_fn=batchify_fn) - symbol, arg_params, aux_params = mx.module.module.load_checkpoint(prefix, epoch) + data_loader = DataLoader(dataset, + batch_size, + last_batch='keep', + num_workers=8, + thread_pool=True, + prefetch=64, + batchify_fn=batchify_fn) + symbol, arg_params, aux_params = mx.module.module.load_checkpoint( + prefix, epoch) # init model list for i in range(num_ctx): model = mx.mod.Module(symbol, context=mx.gpu(i), label_names=None) - model.bind(for_training=False, data_shapes=[('data', (2 * batch_size, 3, 112, 112))]) + model.bind(for_training=False, + data_shapes=[('data', (2 * batch_size, 3, 112, 112))]) model.set_params(arg_params, aux_params) model_list.append(model) # extract parallel and async num_model = len(model_list) for image in tqdm(data_loader): - data_batch = mx.io.DataBatch(data=(image,)) + data_batch = mx.io.DataBatch(data=(image, )) model_list[num_iter % num_model].forward(data_batch, is_train=False) - feat = model_list[num_iter % num_model].get_outputs(merge_multi_context=True)[0] + feat = model_list[num_iter % + num_model].get_outputs(merge_multi_context=True)[0] feat = mx.nd.L2Normalization(feat) feat = mx.nd.reshape(feat, (-1, size * 2)) - feat_mat[ - batch_size * num_iter: - batch_size * num_iter + feat.shape[0], :] = feat.as_in_context(mx.cpu()) + feat_mat[batch_size * num_iter:batch_size * num_iter + + feat.shape[0], :] = feat.as_in_context(mx.cpu()) num_iter += 1 # if num_iter % 20 == 0: # mx.nd.waitall() @@ -180,22 +189,26 @@ def image2template_feature(img_feats=None, templates=None, medias=None): for count_template, uqt in enumerate(unique_templates): - (ind_t,) = np.where(templates == uqt) + (ind_t, ) = np.where(templates == uqt) face_norm_feats = img_feats[ind_t] face_medias = medias[ind_t] - unique_medias, unique_media_counts = np.unique(face_medias, return_counts=True) + unique_medias, unique_media_counts = np.unique(face_medias, + return_counts=True) media_norm_feats = [] for u, ct in zip(unique_medias, unique_media_counts): - (ind_m,) = np.where(face_medias == u) + (ind_m, ) = np.where(face_medias == u) if ct == 1: media_norm_feats += [face_norm_feats[ind_m]] else: # image features from the same video will be aggregated into one feature - media_norm_feats += [np.mean(face_norm_feats[ind_m], axis=0, keepdims=True)] + media_norm_feats += [ + np.mean(face_norm_feats[ind_m], axis=0, keepdims=True) + ] media_norm_feats = np.array(media_norm_feats) # media_norm_feats = media_norm_feats / np.sqrt(np.sum(media_norm_feats ** 2, -1, keepdims=True)) template_feats[count_template] = np.sum(media_norm_feats, axis=0) if count_template % 2000 == 0: - print('Finish Calculating {} template features.'.format(count_template)) + print('Finish Calculating {} template features.'.format( + count_template)) # template_norm_feats = template_feats / np.sqrt(np.sum(template_feats ** 2, -1, keepdims=True)) template_norm_feats = sklearn.preprocessing.normalize(template_feats) # print(template_norm_feats.shape) @@ -205,7 +218,10 @@ def image2template_feature(img_feats=None, templates=None, medias=None): # In[ ]: -def verification(template_norm_feats=None, unique_templates=None, p1=None, p2=None): +def verification(template_norm_feats=None, + unique_templates=None, + p1=None, + p2=None): # ========================================================== # Compute set-to-set Similarity Score. # ========================================================== @@ -213,11 +229,13 @@ def verification(template_norm_feats=None, unique_templates=None, p1=None, p2=No for count_template, uqt in enumerate(unique_templates): template2id[uqt] = count_template - score = np.zeros((len(p1),)) # save cosine distance between pairs + score = np.zeros((len(p1), )) # save cosine distance between pairs total_pairs = np.array(range(len(p1))) batchsize = 100000 # small batchsize instead of all pairs in one batch due to the memory limiation - sublists = [total_pairs[i:i + batchsize] for i in range(0, len(p1), batchsize)] + sublists = [ + total_pairs[i:i + batchsize] for i in range(0, len(p1), batchsize) + ] total_sublists = len(sublists) for c, s in enumerate(sublists): feat1 = template_norm_feats[template2id[p1[s]]] @@ -230,14 +248,19 @@ def verification(template_norm_feats=None, unique_templates=None, p1=None, p2=No # In[ ]: -def verification2(template_norm_feats=None, unique_templates=None, p1=None, p2=None): +def verification2(template_norm_feats=None, + unique_templates=None, + p1=None, + p2=None): template2id = np.zeros((max(unique_templates) + 1, 1), dtype=int) for count_template, uqt in enumerate(unique_templates): template2id[uqt] = count_template - score = np.zeros((len(p1),)) # save cosine distance between pairs + score = np.zeros((len(p1), )) # save cosine distance between pairs total_pairs = np.array(range(len(p1))) batchsize = 100000 # small batchsize instead of all pairs in one batch due to the memory limiation - sublists = [total_pairs[i:i + batchsize] for i in range(0, len(p1), batchsize)] + sublists = [ + total_pairs[i:i + batchsize] for i in range(0, len(p1), batchsize) + ] total_sublists = len(sublists) for c, s in enumerate(sublists): feat1 = template_norm_feats[template2id[p1[s]]] @@ -267,11 +290,11 @@ assert target == 'IJBC' or target == 'IJBB' # ============================================================= start = timeit.default_timer() templates, medias = read_template_media_list( - os.path.join('%s/meta' % image_path, '%s_face_tid_mid.txt' % target.lower())) + os.path.join('%s/meta' % image_path, + '%s_face_tid_mid.txt' % target.lower())) stop = timeit.default_timer() print('Time: %.2f s. ' % (stop - start)) - # ============================================================= # load template pairs for template-to-template verification # tid : template id, label : 1/0 @@ -280,7 +303,8 @@ print('Time: %.2f s. ' % (stop - start)) # ============================================================= start = timeit.default_timer() p1, p2, label = read_template_pair_list( - os.path.join('%s/meta' % image_path, '%s_template_pair_label.txt' % target.lower())) + os.path.join('%s/meta' % image_path, + '%s_template_pair_label.txt' % target.lower())) stop = timeit.default_timer() print('Time: %.2f s. ' % (stop - start)) @@ -297,7 +321,11 @@ img_list_path = '%s/meta/%s_name_5pts_score.txt' % (image_path, target.lower()) img_list = open(img_list_path) files = img_list.readlines() dataset = DatasetIJB(root=img_path, lines=files, align=True) -img_feats = extract_parallel(args.model_prefix, args.model_epoch, dataset, args.batch_size, size=512) +img_feats = extract_parallel(args.model_prefix, + args.model_epoch, + dataset, + args.batch_size, + size=512) faceness_scores = [] for each_line in files: @@ -306,16 +334,15 @@ for each_line in files: faceness_scores = np.array(faceness_scores).astype(np.float32) - stop = timeit.default_timer() print('Time: %.2f s. ' % (stop - start)) -print('Feature Shape: ({} , {}) .'.format(img_feats.shape[0], img_feats.shape[1])) +print('Feature Shape: ({} , {}) .'.format(img_feats.shape[0], + img_feats.shape[1])) # # Step3: Get Template Features # In[ ]: - # ============================================================= # compute template features from image features. # ============================================================= @@ -327,12 +354,12 @@ start = timeit.default_timer() # 1. FaceScore （Feature Norm） # 2. FaceScore （Detector） - if use_flip_test: # concat --- F1 # img_input_feats = img_feats # add --- F2 - img_input_feats = img_feats[:, 0:img_feats.shape[1] // 2] + img_feats[:, img_feats.shape[1] // 2:] + img_input_feats = img_feats[:, 0:img_feats.shape[1] // + 2] + img_feats[:, img_feats.shape[1] // 2:] else: img_input_feats = img_feats[:, 0:img_feats.shape[1] // 2] @@ -340,7 +367,8 @@ if use_norm_score: img_input_feats = img_input_feats else: # normalise features to remove norm information - img_input_feats = img_input_feats / np.sqrt(np.sum(img_input_feats ** 2, -1, keepdims=True)) + img_input_feats = img_input_feats / np.sqrt( + np.sum(img_input_feats**2, -1, keepdims=True)) if use_detector_score: print(img_input_feats.shape, faceness_scores.shape) @@ -349,7 +377,8 @@ if use_detector_score: else: img_input_feats = img_input_feats -template_norm_feats, unique_templates = image2template_feature(img_input_feats, templates, medias) +template_norm_feats, unique_templates = image2template_feature( + img_input_feats, templates, medias) stop = timeit.default_timer() print('Time: %.2f s. ' % (stop - start)) @@ -357,7 +386,6 @@ print('Time: %.2f s. ' % (stop - start)) # In[ ]: - # ============================================================= # compute verification scores between template pairs. # ============================================================= @@ -380,7 +408,6 @@ np.save(score_save_file, score) # In[ ]: - files = [score_save_file] methods = [] scores = [] @@ -390,9 +417,10 @@ for file in files: methods = np.array(methods) scores = dict(zip(methods, scores)) -colours = dict(zip(methods, sample_colours_from_colourmap(methods.shape[0], 'Set2'))) +colours = dict( + zip(methods, sample_colours_from_colourmap(methods.shape[0], 'Set2'))) # x_labels = [1/(10**x) for x in np.linspace(6, 0, 6)] -x_labels = [10 ** -6, 10 ** -5, 10 ** -4, 10 ** -3, 10 ** -2, 10 ** -1] +x_labels = [10**-6, 10**-5, 10**-4, 10**-3, 10**-2, 10**-1] tpr_fpr_table = PrettyTable(['Methods'] + [str(x) for x in x_labels]) fig = plt.figure() for method in methods: @@ -400,16 +428,21 @@ for method in methods: roc_auc = auc(fpr, tpr) fpr = np.flipud(fpr) tpr = np.flipud(tpr) # select largest tpr at same fpr - plt.plot(fpr, tpr, color=colours[method], lw=1, - label=('[%s (AUC = %0.4f %%)]' % (method.split('-')[-1], roc_auc * 100))) + plt.plot(fpr, + tpr, + color=colours[method], + lw=1, + label=('[%s (AUC = %0.4f %%)]' % + (method.split('-')[-1], roc_auc * 100))) tpr_fpr_row = [] tpr_fpr_row.append("%s-%s" % (method, target)) for fpr_iter in np.arange(len(x_labels)): - _, min_index = min(list(zip(abs(fpr - x_labels[fpr_iter]), range(len(fpr))))) + _, min_index = min( + list(zip(abs(fpr - x_labels[fpr_iter]), range(len(fpr))))) # tpr_fpr_row.append('%.4f' % tpr[min_index]) tpr_fpr_row.append('%.2f' % (tpr[min_index] * 100)) tpr_fpr_table.add_row(tpr_fpr_row) -plt.xlim([10 ** -6, 0.1]) +plt.xlim([10**-6, 0.1]) plt.ylim([0.3, 1.0]) plt.grid(linestyle='--', linewidth=1) plt.xticks(x_labels) diff --git a/recognition/partial_fc/mxnet/evaluation/lfw.py b/recognition/partial_fc/mxnet/evaluation/lfw.py index ecf2005..d900d79 100644 --- a/recognition/partial_fc/mxnet/evaluation/lfw.py +++ b/recognition/partial_fc/mxnet/evaluation/lfw.py @@ -2,19 +2,19 @@ """ # MIT License -# +# # Copyright (c) 2016 David Sandberg -# +# # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: -# +# # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. -# +# # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE @@ -38,98 +38,118 @@ import mxnet as mx from mxnet import ndarray as nd - -def calculate_roc(thresholds, embeddings1, embeddings2, actual_issame, nrof_folds=10, pca = 0): - assert(embeddings1.shape[0] == embeddings2.shape[0]) - assert(embeddings1.shape[1] == embeddings2.shape[1]) +def calculate_roc(thresholds, + embeddings1, + embeddings2, + actual_issame, + nrof_folds=10, + pca=0): + assert (embeddings1.shape[0] == embeddings2.shape[0]) + assert (embeddings1.shape[1] == embeddings2.shape[1]) nrof_pairs = min(len(actual_issame), embeddings1.shape[0]) nrof_thresholds = len(thresholds) k_fold = KFold(n_splits=nrof_folds, shuffle=False) - - tprs = np.zeros((nrof_folds,nrof_thresholds)) - fprs = np.zeros((nrof_folds,nrof_thresholds)) + + tprs = np.zeros((nrof_folds, nrof_thresholds)) + fprs = np.zeros((nrof_folds, nrof_thresholds)) accuracy = np.zeros((nrof_folds)) indices = np.arange(nrof_pairs) #print('pca', pca) - - if pca==0: - diff = np.subtract(embeddings1, embeddings2) - dist = np.sum(np.square(diff),1) - + + if pca == 0: + diff = np.subtract(embeddings1, embeddings2) + dist = np.sum(np.square(diff), 1) + for fold_idx, (train_set, test_set) in enumerate(k_fold.split(indices)): #print('train_set', train_set) #print('test_set', test_set) - if pca>0: - print('doing pca on', fold_idx) - embed1_train = embeddings1[train_set] - embed2_train = embeddings2[train_set] - _embed_train = np.concatenate( (embed1_train, embed2_train), axis=0 ) - #print(_embed_train.shape) - pca_model = PCA(n_components=pca) - pca_model.fit(_embed_train) - embed1 = pca_model.transform(embeddings1) - embed2 = pca_model.transform(embeddings2) - embed1 = sklearn.preprocessing.normalize(embed1) - embed2 = sklearn.preprocessing.normalize(embed2) - #print(embed1.shape, embed2.shape) - diff = np.subtract(embed1, embed2) - dist = np.sum(np.square(diff),1) - + if pca > 0: + print('doing pca on', fold_idx) + embed1_train = embeddings1[train_set] + embed2_train = embeddings2[train_set] + _embed_train = np.concatenate((embed1_train, embed2_train), axis=0) + #print(_embed_train.shape) + pca_model = PCA(n_components=pca) + pca_model.fit(_embed_train) + embed1 = pca_model.transform(embeddings1) + embed2 = pca_model.transform(embeddings2) + embed1 = sklearn.preprocessing.normalize(embed1) + embed2 = sklearn.preprocessing.normalize(embed2) + #print(embed1.shape, embed2.shape) + diff = np.subtract(embed1, embed2) + dist = np.sum(np.square(diff), 1) + # Find the best threshold for the fold acc_train = np.zeros((nrof_thresholds)) for threshold_idx, threshold in enumerate(thresholds): - _, _, acc_train[threshold_idx] = calculate_accuracy(threshold, dist[train_set], actual_issame[train_set]) + _, _, acc_train[threshold_idx] = calculate_accuracy( + threshold, dist[train_set], actual_issame[train_set]) best_threshold_index = np.argmax(acc_train) for threshold_idx, threshold in enumerate(thresholds): - tprs[fold_idx,threshold_idx], fprs[fold_idx,threshold_idx], _ = calculate_accuracy(threshold, dist[test_set], actual_issame[test_set]) - _, _, accuracy[fold_idx] = calculate_accuracy(thresholds[best_threshold_index], dist[test_set], actual_issame[test_set]) - - tpr = np.mean(tprs,0) - fpr = np.mean(fprs,0) + tprs[fold_idx, + threshold_idx], fprs[fold_idx, + threshold_idx], _ = calculate_accuracy( + threshold, dist[test_set], + actual_issame[test_set]) + _, _, accuracy[fold_idx] = calculate_accuracy( + thresholds[best_threshold_index], dist[test_set], + actual_issame[test_set]) + + tpr = np.mean(tprs, 0) + fpr = np.mean(fprs, 0) return tpr, fpr, accuracy + def calculate_accuracy(threshold, dist, actual_issame): predict_issame = np.less(dist, threshold) tp = np.sum(np.logical_and(predict_issame, actual_issame)) fp = np.sum(np.logical_and(predict_issame, np.logical_not(actual_issame))) - tn = np.sum(np.logical_and(np.logical_not(predict_issame), np.logical_not(actual_issame))) + tn = np.sum( + np.logical_and(np.logical_not(predict_issame), + np.logical_not(actual_issame))) fn = np.sum(np.logical_and(np.logical_not(predict_issame), actual_issame)) - - tpr = 0 if (tp+fn==0) else float(tp) / float(tp+fn) - fpr = 0 if (fp+tn==0) else float(fp) / float(fp+tn) - acc = float(tp+tn)/dist.size + + tpr = 0 if (tp + fn == 0) else float(tp) / float(tp + fn) + fpr = 0 if (fp + tn == 0) else float(fp) / float(fp + tn) + acc = float(tp + tn) / dist.size return tpr, fpr, acc - -def calculate_val(thresholds, embeddings1, embeddings2, actual_issame, far_target, nrof_folds=10): - assert(embeddings1.shape[0] == embeddings2.shape[0]) - assert(embeddings1.shape[1] == embeddings2.shape[1]) +def calculate_val(thresholds, + embeddings1, + embeddings2, + actual_issame, + far_target, + nrof_folds=10): + assert (embeddings1.shape[0] == embeddings2.shape[0]) + assert (embeddings1.shape[1] == embeddings2.shape[1]) nrof_pairs = min(len(actual_issame), embeddings1.shape[0]) nrof_thresholds = len(thresholds) k_fold = KFold(n_splits=nrof_folds, shuffle=False) - + val = np.zeros(nrof_folds) far = np.zeros(nrof_folds) - + diff = np.subtract(embeddings1, embeddings2) - dist = np.sum(np.square(diff),1) + dist = np.sum(np.square(diff), 1) indices = np.arange(nrof_pairs) - + for fold_idx, (train_set, test_set) in enumerate(k_fold.split(indices)): - + # Find the threshold that gives FAR = far_target far_train = np.zeros(nrof_thresholds) for threshold_idx, threshold in enumerate(thresholds): - _, far_train[threshold_idx] = calculate_val_far(threshold, dist[train_set], actual_issame[train_set]) - if np.max(far_train)>=far_target: + _, far_train[threshold_idx] = calculate_val_far( + threshold, dist[train_set], actual_issame[train_set]) + if np.max(far_train) >= far_target: f = interpolate.interp1d(far_train, thresholds, kind='slinear') threshold = f(far_target) else: threshold = 0.0 - - val[fold_idx], far[fold_idx] = calculate_val_far(threshold, dist[test_set], actual_issame[test_set]) - + + val[fold_idx], far[fold_idx] = calculate_val_far( + threshold, dist[test_set], actual_issame[test_set]) + val_mean = np.mean(val) far_mean = np.mean(far) val_std = np.std(val) @@ -139,49 +159,70 @@ def calculate_val(thresholds, embeddings1, embeddings2, actual_issame, far_targe def calculate_val_far(threshold, dist, actual_issame): predict_issame = np.less(dist, threshold) true_accept = np.sum(np.logical_and(predict_issame, actual_issame)) - false_accept = np.sum(np.logical_and(predict_issame, np.logical_not(actual_issame))) + false_accept = np.sum( + np.logical_and(predict_issame, np.logical_not(actual_issame))) n_same = np.sum(actual_issame) n_diff = np.sum(np.logical_not(actual_issame)) val = float(true_accept) / float(n_same) far = float(false_accept) / float(n_diff) return val, far -def evaluate(embeddings, actual_issame, nrof_folds=10, pca = 0): + +def evaluate(embeddings, actual_issame, nrof_folds=10, pca=0): # Calculate evaluation metrics thresholds = np.arange(0, 4, 0.01) embeddings1 = embeddings[0::2] embeddings2 = embeddings[1::2] - tpr, fpr, accuracy = calculate_roc(thresholds, embeddings1, embeddings2, - np.asarray(actual_issame), nrof_folds=nrof_folds, pca = pca) + tpr, fpr, accuracy = calculate_roc(thresholds, + embeddings1, + embeddings2, + np.asarray(actual_issame), + nrof_folds=nrof_folds, + pca=pca) thresholds = np.arange(0, 4, 0.001) - val, val_std, far = calculate_val(thresholds, embeddings1, embeddings2, - np.asarray(actual_issame), 1e-3, nrof_folds=nrof_folds) + val, val_std, far = calculate_val(thresholds, + embeddings1, + embeddings2, + np.asarray(actual_issame), + 1e-3, + nrof_folds=nrof_folds) return tpr, fpr, accuracy, val, val_std, far + def get_paths(lfw_dir, pairs, file_ext): nrof_skipped_pairs = 0 path_list = [] issame_list = [] for pair in pairs: if len(pair) == 3: - path0 = os.path.join(lfw_dir, pair[0], pair[0] + '_' + '%04d' % int(pair[1])+'.'+file_ext) - path1 = os.path.join(lfw_dir, pair[0], pair[0] + '_' + '%04d' % int(pair[2])+'.'+file_ext) + path0 = os.path.join( + lfw_dir, pair[0], + pair[0] + '_' + '%04d' % int(pair[1]) + '.' + file_ext) + path1 = os.path.join( + lfw_dir, pair[0], + pair[0] + '_' + '%04d' % int(pair[2]) + '.' + file_ext) issame = True elif len(pair) == 4: - path0 = os.path.join(lfw_dir, pair[0], pair[0] + '_' + '%04d' % int(pair[1])+'.'+file_ext) - path1 = os.path.join(lfw_dir, pair[2], pair[2] + '_' + '%04d' % int(pair[3])+'.'+file_ext) + path0 = os.path.join( + lfw_dir, pair[0], + pair[0] + '_' + '%04d' % int(pair[1]) + '.' + file_ext) + path1 = os.path.join( + lfw_dir, pair[2], + pair[2] + '_' + '%04d' % int(pair[3]) + '.' + file_ext) issame = False - if os.path.exists(path0) and os.path.exists(path1): # Only add the pair if both paths exist - path_list += (path0,path1) + if os.path.exists(path0) and os.path.exists( + path1): # Only add the pair if both paths exist + path_list += (path0, path1) issame_list.append(issame) else: print('not exists', path0, path1) nrof_skipped_pairs += 1 - if nrof_skipped_pairs>0: + if nrof_skipped_pairs > 0: print('Skipped %d image pairs' % nrof_skipped_pairs) - + return path_list, issame_list + def read_pairs(pairs_filename): pairs = [] with open(pairs_filename, 'r') as f: @@ -192,88 +233,93 @@ def read_pairs(pairs_filename): def load_dataset(lfw_dir, image_size): - lfw_pairs = read_pairs(os.path.join(lfw_dir, 'pairs.txt')) - lfw_paths, issame_list = get_paths(lfw_dir, lfw_pairs, 'jpg') - lfw_data_list = [] - for flip in [0,1]: - lfw_data = nd.empty((len(lfw_paths), 3, image_size[0], image_size[1])) - lfw_data_list.append(lfw_data) - i = 0 - for path in lfw_paths: - with open(path, 'rb') as fin: - _bin = fin.read() - img = mx.image.imdecode(_bin) - img = nd.transpose(img, axes=(2, 0, 1)) - for flip in [0,1]: - if flip==1: - img = mx.ndarray.flip(data=img, axis=2) - lfw_data_list[flip][i][:] = img - i+=1 - if i%1000==0: - print('loading lfw', i) - print(lfw_data_list[0].shape) - print(lfw_data_list[1].shape) - return (lfw_data_list, issame_list) + lfw_pairs = read_pairs(os.path.join(lfw_dir, 'pairs.txt')) + lfw_paths, issame_list = get_paths(lfw_dir, lfw_pairs, 'jpg') + lfw_data_list = [] + for flip in [0, 1]: + lfw_data = nd.empty((len(lfw_paths), 3, image_size[0], image_size[1])) + lfw_data_list.append(lfw_data) + i = 0 + for path in lfw_paths: + with open(path, 'rb') as fin: + _bin = fin.read() + img = mx.image.imdecode(_bin) + img = nd.transpose(img, axes=(2, 0, 1)) + for flip in [0, 1]: + if flip == 1: + img = mx.ndarray.flip(data=img, axis=2) + lfw_data_list[flip][i][:] = img + i += 1 + if i % 1000 == 0: + print('loading lfw', i) + print(lfw_data_list[0].shape) + print(lfw_data_list[1].shape) + return (lfw_data_list, issame_list) + def test(lfw_set, mx_model, batch_size): - print('testing lfw..') - lfw_data_list = lfw_set[0] - issame_list = lfw_set[1] - model = mx_model - embeddings_list = [] - for i in range( len(lfw_data_list) ): - lfw_data = lfw_data_list[i] - embeddings = None - ba = 0 - while ba= far_target: f = interpolate.interp1d(far_train, thresholds, kind='slinear') threshold = f(far_target) else: threshold = 0.0 - val[fold_idx], far[fold_idx] = calculate_val_far(threshold, dist[test_set], actual_issame[test_set]) + val[fold_idx], far[fold_idx] = calculate_val_far( + threshold, dist[test_set], actual_issame[test_set]) val_mean = np.mean(val) far_mean = np.mean(far) @@ -162,7 +179,8 @@ def calculate_val(thresholds, embeddings1, embeddings2, actual_issame, far_targe def calculate_val_far(threshold, dist, actual_issame): predict_issame = np.less(dist, threshold) true_accept = np.sum(np.logical_and(predict_issame, actual_issame)) - false_accept = np.sum(np.logical_and(predict_issame, np.logical_not(actual_issame))) + false_accept = np.sum( + np.logical_and(predict_issame, np.logical_not(actual_issame))) n_same = np.sum(actual_issame) n_diff = np.sum(np.logical_not(actual_issame)) # print(true_accept, false_accept) @@ -177,11 +195,19 @@ def evaluate(embeddings, actual_issame, nrof_folds=10, pca=0): thresholds = np.arange(0, 4, 0.01) embeddings1 = embeddings[0::2] embeddings2 = embeddings[1::2] - tpr, fpr, accuracy = calculate_roc(thresholds, embeddings1, embeddings2, - np.asarray(actual_issame), nrof_folds=nrof_folds, pca=pca) + tpr, fpr, accuracy = calculate_roc(thresholds, + embeddings1, + embeddings2, + np.asarray(actual_issame), + nrof_folds=nrof_folds, + pca=pca) thresholds = np.arange(0, 4, 0.001) - val, val_std, far = calculate_val(thresholds, embeddings1, embeddings2, - np.asarray(actual_issame), 1e-3, nrof_folds=nrof_folds) + val, val_std, far = calculate_val(thresholds, + embeddings1, + embeddings2, + np.asarray(actual_issame), + 1e-3, + nrof_folds=nrof_folds) return tpr, fpr, accuracy, val, val_std, far @@ -194,7 +220,8 @@ def load_bin(path, image_size): bins, issame_list = pickle.load(f, encoding='bytes') # py3 data_list = [] for flip in [0, 1]: - data = nd.empty((len(issame_list) * 2, 3, image_size[0], image_size[1])) + data = nd.empty( + (len(issame_list) * 2, 3, image_size[0], image_size[1])) data_list.append(data) for i in range(len(issame_list) * 2): _bin = bins[i] @@ -210,7 +237,12 @@ def load_bin(path, image_size): return (data_list, issame_list) -def test(data_set, mx_model, batch_size, nfolds=10, data_extra=None, label_shape=None): +def test(data_set, + mx_model, + batch_size, + nfolds=10, + data_extra=None, + label_shape=None): print('testing verification..') data_list = data_set[0] issame_list = data_set[1] @@ -220,7 +252,7 @@ def test(data_set, mx_model, batch_size, nfolds=10, data_extra=None, label_shape _data_extra = nd.array(data_extra) time_consumed = 0.0 if label_shape is None: - _label = nd.ones((batch_size,)) + _label = nd.ones((batch_size, )) else: _label = nd.ones(label_shape) for i in range(len(data_list)): @@ -234,9 +266,10 @@ def test(data_set, mx_model, batch_size, nfolds=10, data_extra=None, label_shape # print(_data.shape, _label.shape) time0 = datetime.datetime.now() if data_extra is None: - db = mx.io.DataBatch(data=(_data,), label=(_label,)) + db = mx.io.DataBatch(data=(_data, ), label=(_label, )) else: - db = mx.io.DataBatch(data=(_data, _data_extra), label=(_label,)) + db = mx.io.DataBatch(data=(_data, _data_extra), + label=(_label, )) model.forward(db, is_train=False) net_out = model.get_outputs() @@ -287,12 +320,19 @@ def test(data_set, mx_model, batch_size, nfolds=10, data_extra=None, label_shape embeddings = sklearn.preprocessing.normalize(embeddings) print(embeddings.shape) print('infer time', time_consumed) - _, _, accuracy, val, val_std, far = evaluate(embeddings, issame_list, nrof_folds=nfolds) + _, _, accuracy, val, val_std, far = evaluate(embeddings, + issame_list, + nrof_folds=nfolds) acc2, std2 = np.mean(accuracy), np.std(accuracy) return acc1, std1, acc2, std2, _xnorm, embeddings_list -def test_badcase(data_set, mx_model, batch_size, name='', data_extra=None, label_shape=None): +def test_badcase(data_set, + mx_model, + batch_size, + name='', + data_extra=None, + label_shape=None): print('testing verification badcase..') data_list = data_set[0] issame_list = data_set[1] @@ -302,7 +342,7 @@ def test_badcase(data_set, mx_model, batch_size, name='', data_extra=None, label _data_extra = nd.array(data_extra) time_consumed = 0.0 if label_shape is None: - _label = nd.ones((batch_size,)) + _label = nd.ones((batch_size, )) else: _label = nd.ones(label_shape) for i in range(len(data_list)): @@ -316,9 +356,10 @@ def test_badcase(data_set, mx_model, batch_size, name='', data_extra=None, label # print(_data.shape, _label.shape) time0 = datetime.datetime.now() if data_extra is None: - db = mx.io.DataBatch(data=(_data,), label=(_label,)) + db = mx.io.DataBatch(data=(_data, ), label=(_label, )) else: - db = mx.io.DataBatch(data=(_data, _data_extra), label=(_label,)) + db = mx.io.DataBatch(data=(_data, _data_extra), + label=(_label, )) model.forward(db, is_train=False) net_out = model.get_outputs() _embeddings = net_out[0].asnumpy() @@ -364,15 +405,18 @@ def test_badcase(data_set, mx_model, batch_size, name='', data_extra=None, label for threshold_idx, threshold in enumerate(thresholds): p2 = dist[train_set] p3 = actual_issame[train_set] - _, _, acc_train[threshold_idx] = calculate_accuracy(threshold, p2, p3) + _, _, acc_train[threshold_idx] = calculate_accuracy( + threshold, p2, p3) best_threshold_index = np.argmax(acc_train) for threshold_idx, threshold in enumerate(thresholds): - tprs[fold_idx, threshold_idx], fprs[fold_idx, threshold_idx], _ = calculate_accuracy(threshold, - dist[test_set], - actual_issame[ - test_set]) - _, _, accuracy[fold_idx] = calculate_accuracy(thresholds[best_threshold_index], dist[test_set], - actual_issame[test_set]) + tprs[fold_idx, + threshold_idx], fprs[fold_idx, + threshold_idx], _ = calculate_accuracy( + threshold, dist[test_set], + actual_issame[test_set]) + _, _, accuracy[fold_idx] = calculate_accuracy( + thresholds[best_threshold_index], dist[test_set], + actual_issame[test_set]) best_threshold = thresholds[best_threshold_index] for iid in test_set: ida = iid * 2 @@ -383,7 +427,8 @@ def test_badcase(data_set, mx_model, batch_size, name='', data_extra=None, label if not asame: violate *= -1.0 if violate > 0.0: - imga = data[ida].asnumpy().transpose((1, 2, 0))[..., ::-1] # to bgr + imga = data[ida].asnumpy().transpose( + (1, 2, 0))[..., ::-1] # to bgr imgb = data[idb].asnumpy().transpose((1, 2, 0))[..., ::-1] # print(imga.shape, imgb.shape, violate, asame, _dist) if asame: @@ -408,7 +453,8 @@ def test_badcase(data_set, mx_model, batch_size, name='', data_extra=None, label else: threshold = pouts[-1][3] - for item in [(pouts, 'positive(false_negative).png'), (nouts, 'negative(false_positive).png')]: + for item in [(pouts, 'positive(false_negative).png'), + (nouts, 'negative(false_positive).png')]: cols = 4 rows = 8000 outs = item[0] @@ -423,14 +469,21 @@ def test_badcase(data_set, mx_model, batch_size, name='', data_extra=None, label hack = {} if name.startswith('cfp') and item[1].startswith('pos'): - hack = {0: 'manual/238_13.jpg.jpg', 6: 'manual/088_14.jpg.jpg', 10: 'manual/470_14.jpg.jpg', - 25: 'manual/238_13.jpg.jpg', 28: 'manual/143_11.jpg.jpg'} + hack = { + 0: 'manual/238_13.jpg.jpg', + 6: 'manual/088_14.jpg.jpg', + 10: 'manual/470_14.jpg.jpg', + 25: 'manual/238_13.jpg.jpg', + 28: 'manual/143_11.jpg.jpg' + } filename = item[1] if len(name) > 0: filename = name + "_" + filename filename = os.path.join(out_dir, filename) - img = np.zeros((image_shape[0] * rows + 20, image_shape[1] * cols + (cols - 1) * gap, 3), dtype=np.uint8) + img = np.zeros((image_shape[0] * rows + 20, image_shape[1] * cols + + (cols - 1) * gap, 3), + dtype=np.uint8) img[:, :, :] = 255 text_color = (0, 0, 153) text_color = (255, 178, 102) @@ -462,19 +515,27 @@ def test_badcase(data_set, mx_model, batch_size, name='', data_extra=None, label k = "%.3f" % dist # print(k) font = cv2.FONT_HERSHEY_SIMPLEX - cv2.putText(_img, k, (80, image_shape[0] // 2 + 7), font, 0.6, text_color, 2) + cv2.putText(_img, k, (80, image_shape[0] // 2 + 7), font, 0.6, + text_color, 2) # _filename = filename+"_%d.png"%outi # cv2.imwrite(_filename, _img) img[row * image_shape[0]:(row + 1) * image_shape[0], - (col * image_shape[1] + gap * col):((col + 1) * image_shape[1] + gap * col), :] = _img + (col * image_shape[1] + + gap * col):((col + 1) * image_shape[1] + gap * col), :] = _img # threshold = outs[0][3] font = cv2.FONT_HERSHEY_SIMPLEX k = "threshold: %.3f" % threshold - cv2.putText(img, k, (img.shape[1] // 2 - 70, img.shape[0] - 5), font, 0.6, text_color, 2) + cv2.putText(img, k, (img.shape[1] // 2 - 70, img.shape[0] - 5), font, + 0.6, text_color, 2) cv2.imwrite(filename, img) -def dumpR(data_set, mx_model, batch_size, name='', data_extra=None, label_shape=None): +def dumpR(data_set, + mx_model, + batch_size, + name='', + data_extra=None, + label_shape=None): print('dump verification embedding..') data_list = data_set[0] issame_list = data_set[1] @@ -484,7 +545,7 @@ def dumpR(data_set, mx_model, batch_size, name='', data_extra=None, label_shape= _data_extra = nd.array(data_extra) time_consumed = 0.0 if label_shape is None: - _label = nd.ones((batch_size,)) + _label = nd.ones((batch_size, )) else: _label = nd.ones(label_shape) for i in range(len(data_list)): @@ -498,9 +559,10 @@ def dumpR(data_set, mx_model, batch_size, name='', data_extra=None, label_shape= # print(_data.shape, _label.shape) time0 = datetime.datetime.now() if data_extra is None: - db = mx.io.DataBatch(data=(_data,), label=(_label,)) + db = mx.io.DataBatch(data=(_data, ), label=(_label, )) else: - db = mx.io.DataBatch(data=(_data, _data_extra), label=(_label,)) + db = mx.io.DataBatch(data=(_data, _data_extra), + label=(_label, )) model.forward(db, is_train=False) net_out = model.get_outputs() _embeddings = net_out[0].asnumpy() @@ -517,7 +579,9 @@ def dumpR(data_set, mx_model, batch_size, name='', data_extra=None, label_shape= actual_issame = np.asarray(issame_list) outname = os.path.join('temp.bin') with open(outname, 'wb') as f: - pickle.dump((embeddings, issame_list), f, protocol=pickle.HIGHEST_PROTOCOL) + pickle.dump((embeddings, issame_list), + f, + protocol=pickle.HIGHEST_PROTOCOL) if __name__ == '__main__': @@ -525,8 +589,12 @@ if __name__ == '__main__': parser = argparse.ArgumentParser(description='do verification') # general parser.add_argument('--data-dir', default='', help='') - parser.add_argument('--model', default='../model/softmax,50', help='path to load model.') - parser.add_argument('--target', default='lfw,cfp_ff,cfp_fp,agedb_30', help='test targets.') + parser.add_argument('--model', + default='../model/softmax,50', + help='path to load model.') + parser.add_argument('--target', + default='lfw,cfp_ff,cfp_fp,agedb_30', + help='test targets.') parser.add_argument('--gpu', default=0, type=int, help='gpu id') parser.add_argument('--batch-size', default=32, type=int, help='') parser.add_argument('--max', default='', type=str, help='') @@ -572,7 +640,8 @@ if __name__ == '__main__': sym = all_layers['fc1_output'] model = mx.mod.Module(symbol=sym, context=ctx, label_names=None) # model.bind(data_shapes=[('data', (args.batch_size, 3, image_size[0], image_size[1]))], label_shapes=[('softmax_label', (args.batch_size,))]) - model.bind(data_shapes=[('data', (args.batch_size, 3, image_size[0], image_size[1]))]) + model.bind(data_shapes=[('data', (args.batch_size, 3, image_size[0], + image_size[1]))]) model.set_params(arg_params, aux_params) nets.append(model) time_now = datetime.datetime.now() @@ -593,10 +662,13 @@ if __name__ == '__main__': for i in range(len(ver_list)): results = [] for model in nets: - acc1, std1, acc2, std2, xnorm, embeddings_list = test(ver_list[i], model, args.batch_size, args.nfolds) + acc1, std1, acc2, std2, xnorm, embeddings_list = test( + ver_list[i], model, args.batch_size, args.nfolds) print('[%s]XNorm: %f' % (ver_name_list[i], xnorm)) - print('[%s]Accuracy: %1.5f+-%1.5f' % (ver_name_list[i], acc1, std1)) - print('[%s]Accuracy-Flip: %1.5f+-%1.5f' % (ver_name_list[i], acc2, std2)) + print('[%s]Accuracy: %1.5f+-%1.5f' % + (ver_name_list[i], acc1, std1)) + print('[%s]Accuracy-Flip: %1.5f+-%1.5f' % + (ver_name_list[i], acc2, std2)) results.append(acc2) print('Max of [%s] is %1.5f' % (ver_name_list[i], np.max(results))) elif args.mode == 1: diff --git a/recognition/partial_fc/mxnet/image_iter.py b/recognition/partial_fc/mxnet/image_iter.py index 1a004cb..c3a1871 100644 --- a/recognition/partial_fc/mxnet/image_iter.py +++ b/recognition/partial_fc/mxnet/image_iter.py @@ -21,12 +21,22 @@ logger = logging.getLogger() class FaceImageIter(io.DataIter): - - def __init__(self, batch_size, data_shape, path_imgrec=None, - shuffle=False, aug_list=None, mean=None, - rand_mirror=False, cutoff=0, color_jittering=0, + def __init__(self, + batch_size, + data_shape, + path_imgrec=None, + shuffle=False, + aug_list=None, + mean=None, + rand_mirror=False, + cutoff=0, + color_jittering=0, images_filter=0, - data_name='data', label_name='softmax_label', context=0, context_num=1, **kwargs): + data_name='data', + label_name='softmax_label', + context=0, + context_num=1, + **kwargs): super(FaceImageIter, self).__init__() assert path_imgrec self.context = context @@ -34,14 +44,16 @@ class FaceImageIter(io.DataIter): if path_imgrec: logging.info('loading recordio %s...', path_imgrec) path_imgidx = path_imgrec[0:-4] + ".idx" - self.imgrec = recordio.MXIndexedRecordIO(path_imgidx, path_imgrec, 'r') + self.imgrec = recordio.MXIndexedRecordIO(path_imgidx, path_imgrec, + 'r') s = self.imgrec.read_idx(0) header, _ = recordio.unpack(s) if header.flag > 0: self.header0 = (int(header.label[0]), int(header.label[1])) self.imgidx = [] self.id2range = {} - self.seq_identity = range(int(header.label[0]), int(header.label[1])) + self.seq_identity = range(int(header.label[0]), + int(header.label[1])) for identity in self.seq_identity: s = self.imgrec.read_idx(identity) header, _ = recordio.unpack(s) @@ -69,7 +81,7 @@ class FaceImageIter(io.DataIter): self.nd_mean = mx.nd.array(self.mean).reshape((1, 1, 3)) self.check_data_shape(data_shape) - self.provide_data = [(data_name, (batch_size,) + data_shape)] + self.provide_data = [(data_name, (batch_size, ) + data_shape)] self.batch_size = batch_size self.data_shape = data_shape self.shuffle = shuffle @@ -79,12 +91,13 @@ class FaceImageIter(io.DataIter): self.cutoff = cutoff self.color_jittering = color_jittering self.CJA = mx.image.ColorJitterAug(0.125, 0.125, 0.125) - self.provide_label = [(label_name, (batch_size,))] + self.provide_label = [(label_name, (batch_size, ))] self.cur = 0 self.nbatch = 0 self.is_init = False - self.num_samples_per_gpu = int(math.floor(len(self.seq) * 1.0 / self.context_num)) + self.num_samples_per_gpu = int( + math.floor(len(self.seq) * 1.0 / self.context_num)) def reset(self): """Resets the iterator to the beginning of the data.""" @@ -104,7 +117,8 @@ class FaceImageIter(io.DataIter): while True: if self.cur >= self.num_samples_per_gpu: raise StopIteration - idx = self.seq[self.num_samples_per_gpu * self.context + self.cur] + idx = self.seq[self.num_samples_per_gpu * self.context + + self.cur] self.cur += 1 if self.imgrec is not None: s = self.imgrec.read_idx(idx) @@ -242,9 +256,11 @@ class FaceImageIter(io.DataIter): def check_data_shape(self, data_shape): """Checks if the input data shape is valid""" if not len(data_shape) == 3: - raise ValueError('data_shape should have length 3, with dimensions CxHxW') + raise ValueError( + 'data_shape should have length 3, with dimensions CxHxW') if not data_shape[0] == 3: - raise ValueError('This iterator expects inputs to have 3 channels.') + raise ValueError( + 'This iterator expects inputs to have 3 channels.') def check_valid_image(self, data): """Checks if the input data is valid""" @@ -303,17 +319,23 @@ class FaceImageIterList(io.DataIter): # dummy class DummyIter(mx.io.DataIter): - def __init__(self, batch_size, data_shape, batches=1000, mode='', dtype='float32'): + def __init__(self, + batch_size, + data_shape, + batches=1000, + mode='', + dtype='float32'): super(DummyIter, self).__init__(batch_size) - self.data_shape = (batch_size,) + data_shape - self.label_shape = (batch_size,) + self.data_shape = (batch_size, ) + data_shape + self.label_shape = (batch_size, ) self.provide_data = [('data', self.data_shape)] self.provide_label = [('softmax_label', self.label_shape)] # self.provide_label = [('label', self.label_shape)] # if mode == 'perseus': # self.provide_label = [] - self.batch = mx.io.DataBatch(data=[mx.nd.zeros(self.data_shape, dtype=dtype)], - label=[mx.nd.zeros(self.label_shape, dtype=dtype)]) + self.batch = mx.io.DataBatch( + data=[mx.nd.zeros(self.data_shape, dtype=dtype)], + label=[mx.nd.zeros(self.label_shape, dtype=dtype)]) self._batches = 0 self.batches = batches @@ -324,4 +346,3 @@ class DummyIter(mx.io.DataIter): else: self._batches = 0 raise StopIteration - diff --git a/recognition/partial_fc/mxnet/memory_bank.py b/recognition/partial_fc/mxnet/memory_bank.py index 4313eaf..d24a212 100644 --- a/recognition/partial_fc/mxnet/memory_bank.py +++ b/recognition/partial_fc/mxnet/memory_bank.py @@ -8,7 +8,14 @@ from memory_samplers import WeightIndexSampler class MemoryBank(object): - def __init__(self, num_sample, num_local, rank, local_rank, embedding_size, prefix, gpu=True): + def __init__(self, + num_sample, + num_local, + rank, + local_rank, + embedding_size, + prefix, + gpu=True): """ Parameters ---------- @@ -40,13 +47,16 @@ class MemoryBank(object): context = mx.cpu() # In order to apply update, weight and momentum should be storage. - self.weight = nd.random_normal( - loc=0, scale=0.01, shape=(self.num_local, self.embedding_size), - ctx=context) + self.weight = nd.random_normal(loc=0, + scale=0.01, + shape=(self.num_local, + self.embedding_size), + ctx=context) self.weight_mom = nd.zeros_like(self.weight) # Sampler object - self.weight_index_sampler = WeightIndexSampler(num_sample, num_local, rank) + self.weight_index_sampler = WeightIndexSampler(num_sample, num_local, + rank) def sample(self, global_label): """ @@ -99,5 +109,9 @@ class MemoryBank(object): self.weight_mom[index] = updated_weight_mom def save(self): - nd.save(fname=os.path.join(self.prefix, "%d_centers.param" % self.rank), data=self.weight) - nd.save(fname=os.path.join(self.prefix, "%d_centers_mom.param" % self.rank), data=self.weight_mom) + nd.save(fname=os.path.join(self.prefix, + "%d_centers.param" % self.rank), + data=self.weight) + nd.save(fname=os.path.join(self.prefix, + "%d_centers_mom.param" % self.rank), + data=self.weight_mom) diff --git a/recognition/partial_fc/mxnet/memory_module.py b/recognition/partial_fc/mxnet/memory_module.py index 6d9de60..5b85c24 100644 --- a/recognition/partial_fc/mxnet/memory_module.py +++ b/recognition/partial_fc/mxnet/memory_module.py @@ -33,24 +33,33 @@ class SampleDistributeModule(object): The updater of memory bank, default is sgd optimizer. logger: """ - def __init__(self, symbol, fc7_model, memory_bank, memory_optimizer, - logger=logging, ): + def __init__( + self, + symbol, + fc7_model, + memory_bank, + memory_optimizer, + logger=logging, + ): self.size = hvd.size() self.rank = hvd.rank() self.local_rank = hvd.local_rank() self.gpu = mx.gpu(self.local_rank) - self.cpu = mx.cpu() # `device_id` is not needed for CPU. + self.cpu = mx.cpu() # `device_id` is not needed for CPU. self.nd_cache = {} self.embedding_size = config.embedding_size self.batch_size = config.batch_size self.num_update = 0 - self.batch_end_param = namedtuple('batch_end_param', ['loss', 'num_epoch', 'num_update']) + self.batch_end_param = namedtuple('batch_end_param', + ['loss', 'num_epoch', 'num_update']) self.fc7_model = fc7_model self.symbol = symbol self.logger = logger - self.backbone_module = mx.module.Module( - self.symbol, ['data'], ['softmax_label'], logger=self.logger, context=self.gpu) + self.backbone_module = mx.module.Module(self.symbol, ['data'], + ['softmax_label'], + logger=self.logger, + context=self.gpu) self.memory_bank = memory_bank self.memory_optimizer = memory_optimizer @@ -78,8 +87,13 @@ class SampleDistributeModule(object): rank_0_dict[key] = hvd.broadcast(tensor, 0, key) return rank_0_dict - def fit(self, train_data, optimizer_params, batch_end_callback, - initializer, arg_params=None, aux_params=None): + def fit(self, + train_data, + optimizer_params, + batch_end_callback, + initializer, + arg_params=None, + aux_params=None): # Bind -> Init_params -> Init_optimizers self.bind(train_data.provide_data, train_data.provide_label, True) @@ -115,7 +129,9 @@ class SampleDistributeModule(object): if batch_end_callback is not None: batch_end_params = self.batch_end_param( - loss=self.loss_cache, num_epoch=num_epoch, num_update=self.num_update) + loss=self.loss_cache, + num_epoch=num_epoch, + num_update=self.num_update) batch_end_callback(batch_end_params) def get_export_params(self): @@ -143,8 +159,13 @@ class SampleDistributeModule(object): v = self.nd_cache[key] return v - def init_params(self, initializer, arg_params=None, aux_params=None, - allow_missing=False, force_init=False, allow_extra=False): + def init_params(self, + initializer, + arg_params=None, + aux_params=None, + allow_missing=False, + force_init=False, + allow_extra=False): """Initializes the parameters and auxiliary states. Parameters @@ -168,29 +189,34 @@ class SampleDistributeModule(object): contain extra parameters that is not needed by the executor. """ # backbone - self.backbone_module.init_params( - initializer=initializer, arg_params=arg_params, - aux_params=aux_params, allow_missing=allow_missing, - force_init=force_init, allow_extra=allow_extra) + self.backbone_module.init_params(initializer=initializer, + arg_params=arg_params, + aux_params=aux_params, + allow_missing=allow_missing, + force_init=force_init, + allow_extra=allow_extra) def prepare(self, data_batch, sparse_row_id_fn=None): if sparse_row_id_fn is not None: - warnings.warn(UserWarning("sparse_row_id_fn is not invoked for BaseModule.")) + warnings.warn( + UserWarning("sparse_row_id_fn is not invoked for BaseModule.")) def allgather(self, tensor, name, shape, dtype, context): """ Implement in-place AllGather using AllReduce """ - assert isinstance(tensor, nd.NDArray), type(tensor) - assert isinstance(name, str), type(name) - assert isinstance(shape, tuple), type(shape) - assert isinstance(dtype, str), type(dtype) + assert isinstance(tensor, nd.NDArray), type(tensor) + assert isinstance(name, str), type(name) + assert isinstance(shape, tuple), type(shape) + assert isinstance(dtype, str), type(dtype) assert isinstance(context, mx.context.Context), type(context) - total_tensor = self.get_ndarray( - context=context, name=name, shape=shape, dtype=dtype) - total_tensor[:] = 0 # reset array before all-reduce is very important - total_tensor[self.rank * self.batch_size: - self.rank * self.batch_size + self.batch_size] = tensor - hvd.allreduce_(total_tensor, average=False) # all-reduce in-place + total_tensor = self.get_ndarray(context=context, + name=name, + shape=shape, + dtype=dtype) + total_tensor[:] = 0 # reset array before all-reduce is very important + total_tensor[self.rank * self.batch_size:self.rank * self.batch_size + + self.batch_size] = tensor + hvd.allreduce_(total_tensor, average=False) # all-reduce in-place return total_tensor def forward(self, data_batch, is_train=None): @@ -200,29 +226,32 @@ class SampleDistributeModule(object): fc1 = self.backbone_module.get_outputs()[0] label = data_batch.label[0] - total_features = self.allgather( - tensor=fc1, - name='total_feature', - shape=(self.batch_size * self.size, self.embedding_size), - dtype='float32', - context=self.gpu - ) - total_labels = self.allgather( - tensor=label, - name='total_label', - shape=(self.batch_size * self.size,), - dtype='int32', - context=self.cpu - ) + total_features = self.allgather(tensor=fc1, + name='total_feature', + shape=(self.batch_size * self.size, + self.embedding_size), + dtype='float32', + context=self.gpu) + total_labels = self.allgather(tensor=label, + name='total_label', + shape=(self.batch_size * + self.size, ), + dtype='int32', + context=self.cpu) return total_features, total_labels else: return None - def backward_all(self, total_feature, total_label, ): + def backward_all( + self, + total_feature, + total_label, + ): # get memory bank learning rate self.memory_lr = self.memory_optimizer.lr_scheduler(self.num_update) - self.grad_cache = self.get_ndarray(self.gpu, 'grad_cache', total_feature.shape) + self.grad_cache = self.get_ndarray(self.gpu, 'grad_cache', + total_feature.shape) self.loss_cache = self.get_ndarray(self.gpu, 'loss_cache', [1]) self.grad_cache[:] = 0 @@ -238,10 +267,9 @@ class SampleDistributeModule(object): total_feature_grad = grad total_feature_grad = hvd.allreduce(total_feature_grad, average=False) - fc1_grad = total_feature_grad[ - self.batch_size * self.rank: - self.batch_size * self.rank + self.batch_size - ] + fc1_grad = total_feature_grad[self.batch_size * + self.rank:self.batch_size * self.rank + + self.batch_size] self.backbone_module.backward(out_grads=[fc1_grad / self.size]) def get_outputs(self, merge_multi_context=True): @@ -253,7 +281,8 @@ class SampleDistributeModule(object): list of NDArray or list of list of NDArray Output. """ - return self.backbone_module.get_outputs(merge_multi_context=merge_multi_context) + return self.backbone_module.get_outputs( + merge_multi_context=merge_multi_context) def update(self): """ @@ -264,7 +293,9 @@ class SampleDistributeModule(object): mx.nd.waitall() def bind(self, data_shapes, label_shapes=None, for_training=True): - self.backbone_module.bind(data_shapes, label_shapes, for_training=for_training) + self.backbone_module.bind(data_shapes, + label_shapes, + for_training=for_training) def init_optimizer(self, optimizer_params, force_init=False): """ @@ -279,14 +310,18 @@ class SampleDistributeModule(object): Default ``False``, indicating whether we should force re-initializing the optimizer in the case an optimizer is already installed. """ - optimizer_backbone = DistributedOptimizer(mx.optimizer.SGD(**optimizer_params)) - self.backbone_module.init_optimizer('local', optimizer_backbone, force_init=force_init) + optimizer_backbone = DistributedOptimizer( + mx.optimizer.SGD(**optimizer_params)) + self.backbone_module.init_optimizer('local', + optimizer_backbone, + force_init=force_init) def backward(self, total_feature, label): memory_bank = self.memory_bank assert memory_bank.num_local == memory_bank.num_sample, "pass" - _data = self.get_ndarray2(self.gpu, "data_%d" % self.rank, total_feature) + _data = self.get_ndarray2(self.gpu, "data_%d" % self.rank, + total_feature) # Attach grad _data.attach_grad() memory_bank.weight.attach_grad() @@ -294,23 +329,27 @@ class SampleDistributeModule(object): # Convert label _label = self.get_ndarray2(self.gpu, 'label_%d' % self.rank, label) _label = _label - int(self.rank * memory_bank.num_local) - _fc7, _one_hot = self.fc7_model.forward( - _data, memory_bank.weight, mapping_label=_label, depth=memory_bank.num_local) + _fc7, _one_hot = self.fc7_model.forward(_data, + memory_bank.weight, + mapping_label=_label, + depth=memory_bank.num_local) # Sync max max_fc7 = nd.max(_fc7, axis=1, keepdims=True) max_fc7 = nd.reshape(max_fc7, -1) - total_max_fc7 = self.get_ndarray( - context=self.gpu, name='total_max_fc7', - shape=(max_fc7.shape[0], self.size), dtype='float32') + total_max_fc7 = self.get_ndarray(context=self.gpu, + name='total_max_fc7', + shape=(max_fc7.shape[0], self.size), + dtype='float32') total_max_fc7[:] = 0 total_max_fc7[:, self.rank] = max_fc7 hvd.allreduce_(total_max_fc7, average=False) - global_max_fc7 = self.get_ndarray( - context=self.gpu, name='global_max_fc7', - shape=(max_fc7.shape[0], 1), dtype='float32') + global_max_fc7 = self.get_ndarray(context=self.gpu, + name='global_max_fc7', + shape=(max_fc7.shape[0], 1), + dtype='float32') nd.max(total_max_fc7, axis=1, keepdims=True, out=global_max_fc7) # Calculate exp(logits) @@ -339,17 +378,17 @@ class SampleDistributeModule(object): # Update center _weight_grad = memory_bank.weight.grad - self.memory_optimizer.update( - weight=memory_bank.weight, - grad=_weight_grad, - state=memory_bank.weight_mom, - learning_rate=self.memory_lr) + self.memory_optimizer.update(weight=memory_bank.weight, + grad=_weight_grad, + state=memory_bank.weight_mom, + learning_rate=self.memory_lr) return _data.grad, global_loss def backward_sample(self, total_feature, label): this_rank_classes = int(self.memory_bank.num_sample) - local_index, unique_sorted_global_label = self.memory_bank.sample(label) + local_index, unique_sorted_global_label = self.memory_bank.sample( + label) # Get local index _mapping_dict = {} @@ -357,7 +396,8 @@ class SampleDistributeModule(object): global_label_set = set(unique_sorted_global_label) for idx, absolute_label in enumerate(local_sampled_class): if absolute_label in global_label_set: - _mapping_dict[absolute_label] = idx + self.rank * self.memory_bank.num_sample + _mapping_dict[ + absolute_label] = idx + self.rank * self.memory_bank.num_sample label_list = list(label.asnumpy()) mapping_label = [] @@ -377,37 +417,51 @@ class SampleDistributeModule(object): # Sync to gpu if self.memory_bank.gpu: - _data = self.get_ndarray2(self.gpu, "data_%d" % self.rank, total_feature) - _weight = self.get_ndarray2(self.gpu, 'weight_%d' % self.rank, sample_weight) - _weight_mom = self.get_ndarray2(self.gpu, 'weight_mom_%d' % self.rank, sample_weight_mom) + _data = self.get_ndarray2(self.gpu, "data_%d" % self.rank, + total_feature) + _weight = self.get_ndarray2(self.gpu, 'weight_%d' % self.rank, + sample_weight) + _weight_mom = self.get_ndarray2(self.gpu, + 'weight_mom_%d' % self.rank, + sample_weight_mom) else: - _data = self.get_ndarray2(self.gpu, "data_%d" % self.rank, total_feature) - _weight = self.get_ndarray2(self.gpu, 'weight_%d' % self.rank, sample_weight) - _weight_mom = self.get_ndarray2(self.gpu, 'weight_mom_%d' % self.rank, sample_weight_mom) + _data = self.get_ndarray2(self.gpu, "data_%d" % self.rank, + total_feature) + _weight = self.get_ndarray2(self.gpu, 'weight_%d' % self.rank, + sample_weight) + _weight_mom = self.get_ndarray2(self.gpu, + 'weight_mom_%d' % self.rank, + sample_weight_mom) # Attach grad _data.attach_grad() _weight.attach_grad() # Convert label - _label = self.get_ndarray2(self.gpu, 'mapping_label_%d' % self.rank, mapping_label) + _label = self.get_ndarray2(self.gpu, 'mapping_label_%d' % self.rank, + mapping_label) _label = _label - int(self.rank * self.memory_bank.num_sample) - _fc7, _one_hot = self.fc7_model.forward(_data, _weight, mapping_label=_label, depth=this_rank_classes) + _fc7, _one_hot = self.fc7_model.forward(_data, + _weight, + mapping_label=_label, + depth=this_rank_classes) # Sync max max_fc7 = nd.max(_fc7, axis=1, keepdims=True) max_fc7 = nd.reshape(max_fc7, -1) - total_max_fc7 = self.get_ndarray( - context=self.gpu, name='total_max_fc7', - shape=(max_fc7.shape[0], self.size), dtype='float32') + total_max_fc7 = self.get_ndarray(context=self.gpu, + name='total_max_fc7', + shape=(max_fc7.shape[0], self.size), + dtype='float32') total_max_fc7[:] = 0 total_max_fc7[:, self.rank] = max_fc7 hvd.allreduce_(total_max_fc7, average=False) - global_max_fc7 = self.get_ndarray( - context=self.gpu, name='global_max_fc7', - shape=(max_fc7.shape[0], 1), dtype='float32') + global_max_fc7 = self.get_ndarray(context=self.gpu, + name='global_max_fc7', + shape=(max_fc7.shape[0], 1), + dtype='float32') nd.max(total_max_fc7, axis=1, keepdims=True, out=global_max_fc7) # Calculate exp(logits) @@ -435,15 +489,20 @@ class SampleDistributeModule(object): # Update center _weight_grad = _weight.grad - self.memory_optimizer.update(weight=_weight, grad=_weight_grad, state=_weight_mom, learning_rate=self.memory_lr) + self.memory_optimizer.update(weight=_weight, + grad=_weight_grad, + state=_weight_mom, + learning_rate=self.memory_lr) if self.memory_bank.gpu: - self.memory_bank.set( - index=local_index, - updated_weight=_weight, - updated_weight_mom=_weight_mom) + self.memory_bank.set(index=local_index, + updated_weight=_weight, + updated_weight_mom=_weight_mom) else: - self.memory_bank.set( - index=local_index, - updated_weight=self.get_ndarray2(mx.cpu(), "cpu_weight_%d" % self.rank, _weight), - updated_weight_mom=self.get_ndarray2(mx.cpu(), "cpu_weight_mom_%d" % self.rank, _weight_mom)) + self.memory_bank.set(index=local_index, + updated_weight=self.get_ndarray2( + mx.cpu(), "cpu_weight_%d" % self.rank, + _weight), + updated_weight_mom=self.get_ndarray2( + mx.cpu(), "cpu_weight_mom_%d" % self.rank, + _weight_mom)) return _data.grad, global_loss diff --git a/recognition/partial_fc/mxnet/memory_samplers.py b/recognition/partial_fc/mxnet/memory_samplers.py index b0e7b77..ee89491 100644 --- a/recognition/partial_fc/mxnet/memory_samplers.py +++ b/recognition/partial_fc/mxnet/memory_samplers.py @@ -4,7 +4,6 @@ import numpy as np class CenterPositiveClassGet(object): """ Get the corresponding center of the positive class """ - def __init__(self, num_sample, num_local, rank): self.num_sample = num_sample self.num_local = num_local @@ -31,7 +30,6 @@ class CenterPositiveClassGet(object): class CenterNegetiveClassSample(object): """ Sample negative class center """ - def __init__(self, num_sample, num_local, rank): self.num_sample = num_sample self.num_local = num_local @@ -45,9 +43,12 @@ class CenterNegetiveClassSample(object): ------- negative_center_index: list of int """ - negative_class_pool = np.setdiff1d(self.negative_class_pool, positive_center_index) + negative_class_pool = np.setdiff1d(self.negative_class_pool, + positive_center_index) negative_sample_size = self.num_sample - len(positive_center_index) - negative_center_index = np.random.choice(negative_class_pool, negative_sample_size, replace=False) + negative_center_index = np.random.choice(negative_class_pool, + negative_sample_size, + replace=False) return negative_center_index @@ -68,7 +69,9 @@ class WeightIndexSampler(object): positive_center_index = positive_center_index[:self.num_sample] negative_center_index = self.negative(positive_center_index) # - final_center_index = np.concatenate((positive_center_index, negative_center_index)) - assert len(final_center_index) == len(np.unique(final_center_index)) == self.num_sample + final_center_index = np.concatenate( + (positive_center_index, negative_center_index)) + assert len(final_center_index) == len( + np.unique(final_center_index)) == self.num_sample assert len(final_center_index) == self.num_sample return final_center_index diff --git a/recognition/partial_fc/mxnet/memory_scheduler.py b/recognition/partial_fc/mxnet/memory_scheduler.py index b05dd87..2b1b36d 100644 --- a/recognition/partial_fc/mxnet/memory_scheduler.py +++ b/recognition/partial_fc/mxnet/memory_scheduler.py @@ -5,7 +5,7 @@ import mxnet as mx def get_scheduler(): step = [int(x) for x in config.lr_steps.split(',')] backbone_lr_scheduler = mx.lr_scheduler.MultiFactorScheduler( - step=step, factor=0.1, base_lr=config.backbone_lr) + step=step, factor=0.1, base_lr=config.backbone_lr) memory_bank_lr_scheduler = mx.lr_scheduler.MultiFactorScheduler( step=step, factor=0.1, base_lr=config.memory_bank_lr) diff --git a/recognition/partial_fc/mxnet/memory_softmax.py b/recognition/partial_fc/mxnet/memory_softmax.py index 4389872..d64159a 100644 --- a/recognition/partial_fc/mxnet/memory_softmax.py +++ b/recognition/partial_fc/mxnet/memory_softmax.py @@ -25,9 +25,10 @@ class MarginLoss(object): # fc7 = nd.dot(norm_data, norm_weight, transpose_b=True) # - mapping_label_onehot = mx.nd.one_hot( - indices=mapping_label, depth=depth, - on_value=1.0, off_value=0.0) + mapping_label_onehot = mx.nd.one_hot(indices=mapping_label, + depth=depth, + on_value=1.0, + off_value=0.0) # cosface if self.loss_m1 == 1.0 and self.loss_m2 == 0.0: _one_hot = mapping_label_onehot * self.loss_m3 diff --git a/recognition/partial_fc/mxnet/optimizer.py b/recognition/partial_fc/mxnet/optimizer.py index 826acab..b8112e9 100644 --- a/recognition/partial_fc/mxnet/optimizer.py +++ b/recognition/partial_fc/mxnet/optimizer.py @@ -21,8 +21,10 @@ class DistributedOptimizer(mx.optimizer.Optimizer): if isinstance(index, (tuple, list)): for i in range(len(index)): - hvd.allreduce_(grad[i], average=False, - name=self._prefix + str(index[i]), priority=-i) + hvd.allreduce_(grad[i], + average=False, + name=self._prefix + str(index[i]), + priority=-i) else: hvd.allreduce_(grad, average=False, name=self._prefix + str(index)) @@ -58,6 +60,12 @@ class MemoryBankSGDOptimizer(object): if self.momentum > 0: kwargs['momentum'] = self.momentum if state is not None: - nd.sgd_mom_update(weight, grad, state, out=weight, lr=lr, wd=self.wd, **kwargs) + nd.sgd_mom_update(weight, + grad, + state, + out=weight, + lr=lr, + wd=self.wd, + **kwargs) else: raise ValueError diff --git a/recognition/partial_fc/mxnet/symbol/memonger.py b/recognition/partial_fc/mxnet/symbol/memonger.py index d054bda..8ad610b 100644 --- a/recognition/partial_fc/mxnet/symbol/memonger.py +++ b/recognition/partial_fc/mxnet/symbol/memonger.py @@ -1,6 +1,7 @@ import mxnet as mx import math + def prod(shape): """Get product of the shape. """ @@ -138,7 +139,10 @@ def search_plan(sym, ntrial=6, type_dict=None, **kwargs): for k in range(nbegin): info = {} - sym = make_mirror_plan(sym, threshold=threshold, plan_info=info, **kwargs) + sym = make_mirror_plan(sym, + threshold=threshold, + plan_info=info, + **kwargs) cost = get_cost(sym, type_dict, **kwargs) save_size = info['save_size'] >> 20 local_size = info['max_size'] >> 20 @@ -147,7 +151,7 @@ def search_plan(sym, ntrial=6, type_dict=None, **kwargs): min_cost = cost if min_threshold is None or local_size < min_threshold: min_threshold = local_size - print ("Search threshold=%d MB, cost=%d MB" % (threshold, cost)) + print("Search threshold=%d MB, cost=%d MB" % (threshold, cost)) history.append((cost, threshold, sym)) threshold = guess @@ -156,13 +160,16 @@ def search_plan(sym, ntrial=6, type_dict=None, **kwargs): threshold = min_threshold + step if step > 0: for k in range(ntrial): - sym = make_mirror_plan(sym, threshold=threshold, plan_info=info, **kwargs) + sym = make_mirror_plan(sym, + threshold=threshold, + plan_info=info, + **kwargs) cost = get_cost(sym, type_dict, **kwargs) - print ("Search threshold=%d MB, cost=%d MB" % (threshold, cost)) + print("Search threshold=%d MB, cost=%d MB" % (threshold, cost)) history.append((cost, threshold, sym)) threshold += step - history.sort(key = lambda x: x[0]) + history.sort(key=lambda x: x[0]) cost, threshold, sym = history[0] print('Find best plan with threshold=%d, cost=%d MB' % (threshold, cost)) return sym diff --git a/recognition/partial_fc/mxnet/symbol/resnet.py b/recognition/partial_fc/mxnet/symbol/resnet.py index 231f4c9..bd86106 100644 --- a/recognition/partial_fc/mxnet/symbol/resnet.py +++ b/recognition/partial_fc/mxnet/symbol/resnet.py @@ -14,7 +14,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - ''' Adapted from https://github.com/tornadomeet/ResNet/blob/master/symbol_resnet.py Original author Wei Wu @@ -55,7 +54,8 @@ def Act(data, act_type, name): return body -def residual_unit_v1(data, num_filter, stride, dim_match, name, bottle_neck, **kwargs): +def residual_unit_v1(data, num_filter, stride, dim_match, name, bottle_neck, + **kwargs): """Return ResNet Unit symbol for building ResNet Parameters ---------- @@ -81,71 +81,176 @@ def residual_unit_v1(data, num_filter, stride, dim_match, name, bottle_neck, **k act_type = kwargs.get('version_act', 'prelu') # print('in unit1') if bottle_neck: - conv1 = Conv(data=data, num_filter=int(num_filter * 0.25), kernel=(1, 1), stride=stride, pad=(0, 0), - no_bias=True, workspace=workspace, name=name + '_conv1') - bn1 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn1') + conv1 = Conv(data=data, + num_filter=int(num_filter * 0.25), + kernel=(1, 1), + stride=stride, + pad=(0, 0), + no_bias=True, + workspace=workspace, + name=name + '_conv1') + bn1 = mx.sym.BatchNorm(data=conv1, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn1') act1 = Act(data=bn1, act_type=act_type, name=name + '_relu1') - conv2 = Conv(data=act1, num_filter=int(num_filter * 0.25), kernel=(3, 3), stride=(1, 1), pad=(1, 1), - no_bias=True, workspace=workspace, name=name + '_conv2') - bn2 = mx.sym.BatchNorm(data=conv2, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn2') + conv2 = Conv(data=act1, + num_filter=int(num_filter * 0.25), + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv2') + bn2 = mx.sym.BatchNorm(data=conv2, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn2') act2 = Act(data=bn2, act_type=act_type, name=name + '_relu2') - conv3 = Conv(data=act2, num_filter=num_filter, kernel=(1, 1), stride=(1, 1), pad=(0, 0), no_bias=True, - workspace=workspace, name=name + '_conv3') - bn3 = mx.sym.BatchNorm(data=conv3, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn3') + conv3 = Conv(data=act2, + num_filter=num_filter, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + no_bias=True, + workspace=workspace, + name=name + '_conv3') + bn3 = mx.sym.BatchNorm(data=conv3, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn3') if use_se: # se begin - body = mx.sym.Pooling(data=bn3, global_pool=True, kernel=(7, 7), pool_type='avg', name=name + '_se_pool1') - body = Conv(data=body, num_filter=num_filter // 16, kernel=(1, 1), stride=(1, 1), pad=(0, 0), - name=name + "_se_conv1", workspace=workspace) + body = mx.sym.Pooling(data=bn3, + global_pool=True, + kernel=(7, 7), + pool_type='avg', + name=name + '_se_pool1') + body = Conv(data=body, + num_filter=num_filter // 16, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv1", + workspace=workspace) body = Act(data=body, act_type=act_type, name=name + '_se_relu1') - body = Conv(data=body, num_filter=num_filter, kernel=(1, 1), stride=(1, 1), pad=(0, 0), - name=name + "_se_conv2", workspace=workspace) - body = mx.symbol.Activation(data=body, act_type='sigmoid', name=name + "_se_sigmoid") + body = Conv(data=body, + num_filter=num_filter, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv2", + workspace=workspace) + body = mx.symbol.Activation(data=body, + act_type='sigmoid', + name=name + "_se_sigmoid") bn3 = mx.symbol.broadcast_mul(bn3, body) # se end if dim_match: shortcut = data else: - conv1sc = Conv(data=data, num_filter=num_filter, kernel=(1, 1), stride=stride, no_bias=True, - workspace=workspace, name=name + '_conv1sc') - shortcut = mx.sym.BatchNorm(data=conv1sc, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_sc') + conv1sc = Conv(data=data, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + no_bias=True, + workspace=workspace, + name=name + '_conv1sc') + shortcut = mx.sym.BatchNorm(data=conv1sc, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_sc') if memonger: shortcut._set_attr(mirror_stage='True') - return Act(data=bn3 + shortcut, act_type=act_type, name=name + '_relu3') + return Act(data=bn3 + shortcut, + act_type=act_type, + name=name + '_relu3') else: - conv1 = Conv(data=data, num_filter=num_filter, kernel=(3, 3), stride=stride, pad=(1, 1), - no_bias=True, workspace=workspace, name=name + '_conv1') - bn1 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_bn1') + conv1 = Conv(data=data, + num_filter=num_filter, + kernel=(3, 3), + stride=stride, + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv1') + bn1 = mx.sym.BatchNorm(data=conv1, + fix_gamma=False, + momentum=bn_mom, + eps=2e-5, + name=name + '_bn1') act1 = Act(data=bn1, act_type=act_type, name=name + '_relu1') - conv2 = Conv(data=act1, num_filter=num_filter, kernel=(3, 3), stride=(1, 1), pad=(1, 1), - no_bias=True, workspace=workspace, name=name + '_conv2') - bn2 = mx.sym.BatchNorm(data=conv2, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_bn2') + conv2 = Conv(data=act1, + num_filter=num_filter, + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv2') + bn2 = mx.sym.BatchNorm(data=conv2, + fix_gamma=False, + momentum=bn_mom, + eps=2e-5, + name=name + '_bn2') if use_se: # se begin - body = mx.sym.Pooling(data=bn2, global_pool=True, kernel=(7, 7), pool_type='avg', name=name + '_se_pool1') - body = Conv(data=body, num_filter=num_filter // 16, kernel=(1, 1), stride=(1, 1), pad=(0, 0), - name=name + "_se_conv1", workspace=workspace) + body = mx.sym.Pooling(data=bn2, + global_pool=True, + kernel=(7, 7), + pool_type='avg', + name=name + '_se_pool1') + body = Conv(data=body, + num_filter=num_filter // 16, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv1", + workspace=workspace) body = Act(data=body, act_type=act_type, name=name + '_se_relu1') - body = Conv(data=body, num_filter=num_filter, kernel=(1, 1), stride=(1, 1), pad=(0, 0), - name=name + "_se_conv2", workspace=workspace) - body = mx.symbol.Activation(data=body, act_type='sigmoid', name=name + "_se_sigmoid") + body = Conv(data=body, + num_filter=num_filter, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv2", + workspace=workspace) + body = mx.symbol.Activation(data=body, + act_type='sigmoid', + name=name + "_se_sigmoid") bn2 = mx.symbol.broadcast_mul(bn2, body) # se end if dim_match: shortcut = data else: - conv1sc = Conv(data=data, num_filter=num_filter, kernel=(1, 1), stride=stride, no_bias=True, - workspace=workspace, name=name + '_conv1sc') - shortcut = mx.sym.BatchNorm(data=conv1sc, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_sc') + conv1sc = Conv(data=data, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + no_bias=True, + workspace=workspace, + name=name + '_conv1sc') + shortcut = mx.sym.BatchNorm(data=conv1sc, + fix_gamma=False, + momentum=bn_mom, + eps=2e-5, + name=name + '_sc') if memonger: shortcut._set_attr(mirror_stage='True') - return Act(data=bn2 + shortcut, act_type=act_type, name=name + '_relu3') + return Act(data=bn2 + shortcut, + act_type=act_type, + name=name + '_relu3') -def residual_unit_v1_L(data, num_filter, stride, dim_match, name, bottle_neck, **kwargs): +def residual_unit_v1_L(data, num_filter, stride, dim_match, name, bottle_neck, + **kwargs): """Return ResNet Unit symbol for building ResNet Parameters ---------- @@ -171,71 +276,176 @@ def residual_unit_v1_L(data, num_filter, stride, dim_match, name, bottle_neck, * act_type = kwargs.get('version_act', 'prelu') # print('in unit1') if bottle_neck: - conv1 = Conv(data=data, num_filter=int(num_filter * 0.25), kernel=(1, 1), stride=(1, 1), pad=(0, 0), - no_bias=True, workspace=workspace, name=name + '_conv1') - bn1 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn1') + conv1 = Conv(data=data, + num_filter=int(num_filter * 0.25), + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + no_bias=True, + workspace=workspace, + name=name + '_conv1') + bn1 = mx.sym.BatchNorm(data=conv1, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn1') act1 = Act(data=bn1, act_type=act_type, name=name + '_relu1') - conv2 = Conv(data=act1, num_filter=int(num_filter * 0.25), kernel=(3, 3), stride=(1, 1), pad=(1, 1), - no_bias=True, workspace=workspace, name=name + '_conv2') - bn2 = mx.sym.BatchNorm(data=conv2, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn2') + conv2 = Conv(data=act1, + num_filter=int(num_filter * 0.25), + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv2') + bn2 = mx.sym.BatchNorm(data=conv2, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn2') act2 = Act(data=bn2, act_type=act_type, name=name + '_relu2') - conv3 = Conv(data=act2, num_filter=num_filter, kernel=(1, 1), stride=stride, pad=(0, 0), no_bias=True, - workspace=workspace, name=name + '_conv3') - bn3 = mx.sym.BatchNorm(data=conv3, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn3') + conv3 = Conv(data=act2, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + pad=(0, 0), + no_bias=True, + workspace=workspace, + name=name + '_conv3') + bn3 = mx.sym.BatchNorm(data=conv3, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn3') if use_se: # se begin - body = mx.sym.Pooling(data=bn3, global_pool=True, kernel=(7, 7), pool_type='avg', name=name + '_se_pool1') - body = Conv(data=body, num_filter=num_filter // 16, kernel=(1, 1), stride=(1, 1), pad=(0, 0), - name=name + "_se_conv1", workspace=workspace) + body = mx.sym.Pooling(data=bn3, + global_pool=True, + kernel=(7, 7), + pool_type='avg', + name=name + '_se_pool1') + body = Conv(data=body, + num_filter=num_filter // 16, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv1", + workspace=workspace) body = Act(data=body, act_type=act_type, name=name + '_se_relu1') - body = Conv(data=body, num_filter=num_filter, kernel=(1, 1), stride=(1, 1), pad=(0, 0), - name=name + "_se_conv2", workspace=workspace) - body = mx.symbol.Activation(data=body, act_type='sigmoid', name=name + "_se_sigmoid") + body = Conv(data=body, + num_filter=num_filter, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv2", + workspace=workspace) + body = mx.symbol.Activation(data=body, + act_type='sigmoid', + name=name + "_se_sigmoid") bn3 = mx.symbol.broadcast_mul(bn3, body) # se end if dim_match: shortcut = data else: - conv1sc = Conv(data=data, num_filter=num_filter, kernel=(1, 1), stride=stride, no_bias=True, - workspace=workspace, name=name + '_conv1sc') - shortcut = mx.sym.BatchNorm(data=conv1sc, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_sc') + conv1sc = Conv(data=data, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + no_bias=True, + workspace=workspace, + name=name + '_conv1sc') + shortcut = mx.sym.BatchNorm(data=conv1sc, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_sc') if memonger: shortcut._set_attr(mirror_stage='True') - return Act(data=bn3 + shortcut, act_type=act_type, name=name + '_relu3') + return Act(data=bn3 + shortcut, + act_type=act_type, + name=name + '_relu3') else: - conv1 = Conv(data=data, num_filter=num_filter, kernel=(3, 3), stride=(1, 1), pad=(1, 1), - no_bias=True, workspace=workspace, name=name + '_conv1') - bn1 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_bn1') + conv1 = Conv(data=data, + num_filter=num_filter, + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv1') + bn1 = mx.sym.BatchNorm(data=conv1, + fix_gamma=False, + momentum=bn_mom, + eps=2e-5, + name=name + '_bn1') act1 = Act(data=bn1, act_type=act_type, name=name + '_relu1') - conv2 = Conv(data=act1, num_filter=num_filter, kernel=(3, 3), stride=stride, pad=(1, 1), - no_bias=True, workspace=workspace, name=name + '_conv2') - bn2 = mx.sym.BatchNorm(data=conv2, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_bn2') + conv2 = Conv(data=act1, + num_filter=num_filter, + kernel=(3, 3), + stride=stride, + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv2') + bn2 = mx.sym.BatchNorm(data=conv2, + fix_gamma=False, + momentum=bn_mom, + eps=2e-5, + name=name + '_bn2') if use_se: # se begin - body = mx.sym.Pooling(data=bn2, global_pool=True, kernel=(7, 7), pool_type='avg', name=name + '_se_pool1') - body = Conv(data=body, num_filter=num_filter // 16, kernel=(1, 1), stride=(1, 1), pad=(0, 0), - name=name + "_se_conv1", workspace=workspace) + body = mx.sym.Pooling(data=bn2, + global_pool=True, + kernel=(7, 7), + pool_type='avg', + name=name + '_se_pool1') + body = Conv(data=body, + num_filter=num_filter // 16, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv1", + workspace=workspace) body = Act(data=body, act_type=act_type, name=name + '_se_relu1') - body = Conv(data=body, num_filter=num_filter, kernel=(1, 1), stride=(1, 1), pad=(0, 0), - name=name + "_se_conv2", workspace=workspace) - body = mx.symbol.Activation(data=body, act_type='sigmoid', name=name + "_se_sigmoid") + body = Conv(data=body, + num_filter=num_filter, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv2", + workspace=workspace) + body = mx.symbol.Activation(data=body, + act_type='sigmoid', + name=name + "_se_sigmoid") bn2 = mx.symbol.broadcast_mul(bn2, body) # se end if dim_match: shortcut = data else: - conv1sc = Conv(data=data, num_filter=num_filter, kernel=(1, 1), stride=stride, no_bias=True, - workspace=workspace, name=name + '_conv1sc') - shortcut = mx.sym.BatchNorm(data=conv1sc, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_sc') + conv1sc = Conv(data=data, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + no_bias=True, + workspace=workspace, + name=name + '_conv1sc') + shortcut = mx.sym.BatchNorm(data=conv1sc, + fix_gamma=False, + momentum=bn_mom, + eps=2e-5, + name=name + '_sc') if memonger: shortcut._set_attr(mirror_stage='True') - return Act(data=bn2 + shortcut, act_type=act_type, name=name + '_relu3') + return Act(data=bn2 + shortcut, + act_type=act_type, + name=name + '_relu3') -def residual_unit_v2(data, num_filter, stride, dim_match, name, bottle_neck, **kwargs): +def residual_unit_v2(data, num_filter, stride, dim_match, name, bottle_neck, + **kwargs): """Return ResNet Unit symbol for building ResNet Parameters ---------- @@ -262,66 +472,159 @@ def residual_unit_v2(data, num_filter, stride, dim_match, name, bottle_neck, **k # print('in unit2') if bottle_neck: # the same as https://github.com/facebook/fb.resnet.torch#notes, a bit difference with origin paper - bn1 = mx.sym.BatchNorm(data=data, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn1') + bn1 = mx.sym.BatchNorm(data=data, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn1') act1 = Act(data=bn1, act_type=act_type, name=name + '_relu1') - conv1 = Conv(data=act1, num_filter=int(num_filter * 0.25), kernel=(1, 1), stride=(1, 1), pad=(0, 0), - no_bias=True, workspace=workspace, name=name + '_conv1') - bn2 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn2') + conv1 = Conv(data=act1, + num_filter=int(num_filter * 0.25), + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + no_bias=True, + workspace=workspace, + name=name + '_conv1') + bn2 = mx.sym.BatchNorm(data=conv1, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn2') act2 = Act(data=bn2, act_type=act_type, name=name + '_relu2') - conv2 = Conv(data=act2, num_filter=int(num_filter * 0.25), kernel=(3, 3), stride=stride, pad=(1, 1), - no_bias=True, workspace=workspace, name=name + '_conv2') - bn3 = mx.sym.BatchNorm(data=conv2, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn3') + conv2 = Conv(data=act2, + num_filter=int(num_filter * 0.25), + kernel=(3, 3), + stride=stride, + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv2') + bn3 = mx.sym.BatchNorm(data=conv2, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn3') act3 = Act(data=bn3, act_type=act_type, name=name + '_relu3') - conv3 = Conv(data=act3, num_filter=num_filter, kernel=(1, 1), stride=(1, 1), pad=(0, 0), no_bias=True, - workspace=workspace, name=name + '_conv3') + conv3 = Conv(data=act3, + num_filter=num_filter, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + no_bias=True, + workspace=workspace, + name=name + '_conv3') if use_se: # se begin - body = mx.sym.Pooling(data=conv3, global_pool=True, kernel=(7, 7), pool_type='avg', name=name + '_se_pool1') - body = Conv(data=body, num_filter=num_filter // 16, kernel=(1, 1), stride=(1, 1), pad=(0, 0), - name=name + "_se_conv1", workspace=workspace) + body = mx.sym.Pooling(data=conv3, + global_pool=True, + kernel=(7, 7), + pool_type='avg', + name=name + '_se_pool1') + body = Conv(data=body, + num_filter=num_filter // 16, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv1", + workspace=workspace) body = Act(data=body, act_type=act_type, name=name + '_se_relu1') - body = Conv(data=body, num_filter=num_filter, kernel=(1, 1), stride=(1, 1), pad=(0, 0), - name=name + "_se_conv2", workspace=workspace) - body = mx.symbol.Activation(data=body, act_type='sigmoid', name=name + "_se_sigmoid") + body = Conv(data=body, + num_filter=num_filter, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv2", + workspace=workspace) + body = mx.symbol.Activation(data=body, + act_type='sigmoid', + name=name + "_se_sigmoid") conv3 = mx.symbol.broadcast_mul(conv3, body) if dim_match: shortcut = data else: - shortcut = Conv(data=act1, num_filter=num_filter, kernel=(1, 1), stride=stride, no_bias=True, - workspace=workspace, name=name + '_sc') + shortcut = Conv(data=act1, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + no_bias=True, + workspace=workspace, + name=name + '_sc') if memonger: shortcut._set_attr(mirror_stage='True') return conv3 + shortcut else: - bn1 = mx.sym.BatchNorm(data=data, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_bn1') + bn1 = mx.sym.BatchNorm(data=data, + fix_gamma=False, + momentum=bn_mom, + eps=2e-5, + name=name + '_bn1') act1 = Act(data=bn1, act_type=act_type, name=name + '_relu1') - conv1 = Conv(data=act1, num_filter=num_filter, kernel=(3, 3), stride=stride, pad=(1, 1), - no_bias=True, workspace=workspace, name=name + '_conv1') - bn2 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_bn2') + conv1 = Conv(data=act1, + num_filter=num_filter, + kernel=(3, 3), + stride=stride, + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv1') + bn2 = mx.sym.BatchNorm(data=conv1, + fix_gamma=False, + momentum=bn_mom, + eps=2e-5, + name=name + '_bn2') act2 = Act(data=bn2, act_type=act_type, name=name + '_relu2') - conv2 = Conv(data=act2, num_filter=num_filter, kernel=(3, 3), stride=(1, 1), pad=(1, 1), - no_bias=True, workspace=workspace, name=name + '_conv2') + conv2 = Conv(data=act2, + num_filter=num_filter, + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv2') if use_se: # se begin - body = mx.sym.Pooling(data=conv2, global_pool=True, kernel=(7, 7), pool_type='avg', name=name + '_se_pool1') - body = Conv(data=body, num_filter=num_filter // 16, kernel=(1, 1), stride=(1, 1), pad=(0, 0), - name=name + "_se_conv1", workspace=workspace) + body = mx.sym.Pooling(data=conv2, + global_pool=True, + kernel=(7, 7), + pool_type='avg', + name=name + '_se_pool1') + body = Conv(data=body, + num_filter=num_filter // 16, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv1", + workspace=workspace) body = Act(data=body, act_type=act_type, name=name + '_se_relu1') - body = Conv(data=body, num_filter=num_filter, kernel=(1, 1), stride=(1, 1), pad=(0, 0), - name=name + "_se_conv2", workspace=workspace) - body = mx.symbol.Activation(data=body, act_type='sigmoid', name=name + "_se_sigmoid") + body = Conv(data=body, + num_filter=num_filter, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv2", + workspace=workspace) + body = mx.symbol.Activation(data=body, + act_type='sigmoid', + name=name + "_se_sigmoid") conv2 = mx.symbol.broadcast_mul(conv2, body) if dim_match: shortcut = data else: - shortcut = Conv(data=act1, num_filter=num_filter, kernel=(1, 1), stride=stride, no_bias=True, - workspace=workspace, name=name + '_sc') + shortcut = Conv(data=act1, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + no_bias=True, + workspace=workspace, + name=name + '_sc') if memonger: shortcut._set_attr(mirror_stage='True') return conv2 + shortcut -def residual_unit_v3(data, num_filter, stride, dim_match, name, bottle_neck, **kwargs): +def residual_unit_v3(data, num_filter, stride, dim_match, name, bottle_neck, + **kwargs): """Return ResNet Unit symbol for building ResNet Parameters ---------- @@ -347,73 +650,182 @@ def residual_unit_v3(data, num_filter, stride, dim_match, name, bottle_neck, **k act_type = kwargs.get('version_act', 'prelu') # print('in unit3') if bottle_neck: - bn1 = mx.sym.BatchNorm(data=data, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn1') - conv1 = Conv(data=bn1, num_filter=int(num_filter * 0.25), kernel=(1, 1), stride=(1, 1), pad=(0, 0), - no_bias=True, workspace=workspace, name=name + '_conv1') - bn2 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn2') + bn1 = mx.sym.BatchNorm(data=data, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn1') + conv1 = Conv(data=bn1, + num_filter=int(num_filter * 0.25), + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + no_bias=True, + workspace=workspace, + name=name + '_conv1') + bn2 = mx.sym.BatchNorm(data=conv1, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn2') act1 = Act(data=bn2, act_type=act_type, name=name + '_relu1') - conv2 = Conv(data=act1, num_filter=int(num_filter * 0.25), kernel=(3, 3), stride=(1, 1), pad=(1, 1), - no_bias=True, workspace=workspace, name=name + '_conv2') - bn3 = mx.sym.BatchNorm(data=conv2, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn3') + conv2 = Conv(data=act1, + num_filter=int(num_filter * 0.25), + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv2') + bn3 = mx.sym.BatchNorm(data=conv2, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn3') act2 = Act(data=bn3, act_type=act_type, name=name + '_relu2') - conv3 = Conv(data=act2, num_filter=num_filter, kernel=(1, 1), stride=stride, pad=(0, 0), no_bias=True, - workspace=workspace, name=name + '_conv3') - bn4 = mx.sym.BatchNorm(data=conv3, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn4') + conv3 = Conv(data=act2, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + pad=(0, 0), + no_bias=True, + workspace=workspace, + name=name + '_conv3') + bn4 = mx.sym.BatchNorm(data=conv3, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn4') if use_se: # se begin - body = mx.sym.Pooling(data=bn4, global_pool=True, kernel=(7, 7), pool_type='avg', name=name + '_se_pool1') - body = Conv(data=body, num_filter=num_filter // 16, kernel=(1, 1), stride=(1, 1), pad=(0, 0), - name=name + "_se_conv1", workspace=workspace) + body = mx.sym.Pooling(data=bn4, + global_pool=True, + kernel=(7, 7), + pool_type='avg', + name=name + '_se_pool1') + body = Conv(data=body, + num_filter=num_filter // 16, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv1", + workspace=workspace) body = Act(data=body, act_type=act_type, name=name + '_se_relu1') - body = Conv(data=body, num_filter=num_filter, kernel=(1, 1), stride=(1, 1), pad=(0, 0), - name=name + "_se_conv2", workspace=workspace) - body = mx.symbol.Activation(data=body, act_type='sigmoid', name=name + "_se_sigmoid") + body = Conv(data=body, + num_filter=num_filter, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv2", + workspace=workspace) + body = mx.symbol.Activation(data=body, + act_type='sigmoid', + name=name + "_se_sigmoid") bn4 = mx.symbol.broadcast_mul(bn4, body) # se end if dim_match: shortcut = data else: - conv1sc = Conv(data=data, num_filter=num_filter, kernel=(1, 1), stride=stride, no_bias=True, - workspace=workspace, name=name + '_conv1sc') - shortcut = mx.sym.BatchNorm(data=conv1sc, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_sc') + conv1sc = Conv(data=data, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + no_bias=True, + workspace=workspace, + name=name + '_conv1sc') + shortcut = mx.sym.BatchNorm(data=conv1sc, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_sc') if memonger: shortcut._set_attr(mirror_stage='True') return bn4 + shortcut else: - bn1 = mx.sym.BatchNorm(data=data, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn1') - conv1 = Conv(data=bn1, num_filter=num_filter, kernel=(3, 3), stride=(1, 1), pad=(1, 1), - no_bias=True, workspace=workspace, name=name + '_conv1') - bn2 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn2') + bn1 = mx.sym.BatchNorm(data=data, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn1') + conv1 = Conv(data=bn1, + num_filter=num_filter, + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv1') + bn2 = mx.sym.BatchNorm(data=conv1, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn2') act1 = Act(data=bn2, act_type=act_type, name=name + '_relu1') - conv2 = Conv(data=act1, num_filter=num_filter, kernel=(3, 3), stride=stride, pad=(1, 1), - no_bias=True, workspace=workspace, name=name + '_conv2') - bn3 = mx.sym.BatchNorm(data=conv2, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn3') + conv2 = Conv(data=act1, + num_filter=num_filter, + kernel=(3, 3), + stride=stride, + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv2') + bn3 = mx.sym.BatchNorm(data=conv2, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn3') if use_se: # se begin - body = mx.sym.Pooling(data=bn3, global_pool=True, kernel=(7, 7), pool_type='avg', name=name + '_se_pool1') - body = Conv(data=body, num_filter=num_filter // 16, kernel=(1, 1), stride=(1, 1), pad=(0, 0), - name=name + "_se_conv1", workspace=workspace) + body = mx.sym.Pooling(data=bn3, + global_pool=True, + kernel=(7, 7), + pool_type='avg', + name=name + '_se_pool1') + body = Conv(data=body, + num_filter=num_filter // 16, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv1", + workspace=workspace) body = Act(data=body, act_type=act_type, name=name + '_se_relu1') - body = Conv(data=body, num_filter=num_filter, kernel=(1, 1), stride=(1, 1), pad=(0, 0), - name=name + "_se_conv2", workspace=workspace) - body = mx.symbol.Activation(data=body, act_type='sigmoid', name=name + "_se_sigmoid") + body = Conv(data=body, + num_filter=num_filter, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv2", + workspace=workspace) + body = mx.symbol.Activation(data=body, + act_type='sigmoid', + name=name + "_se_sigmoid") bn3 = mx.symbol.broadcast_mul(bn3, body) # se end if dim_match: shortcut = data else: - conv1sc = Conv(data=data, num_filter=num_filter, kernel=(1, 1), stride=stride, no_bias=True, - workspace=workspace, name=name + '_conv1sc') - shortcut = mx.sym.BatchNorm(data=conv1sc, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_sc') + conv1sc = Conv(data=data, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + no_bias=True, + workspace=workspace, + name=name + '_conv1sc') + shortcut = mx.sym.BatchNorm(data=conv1sc, + fix_gamma=False, + momentum=bn_mom, + eps=2e-5, + name=name + '_sc') if memonger: shortcut._set_attr(mirror_stage='True') return bn3 + shortcut -def residual_unit_v3_x(data, num_filter, stride, dim_match, name, bottle_neck, **kwargs): +def residual_unit_v3_x(data, num_filter, stride, dim_match, name, bottle_neck, + **kwargs): """Return ResNeXt Unit symbol for building ResNeXt Parameters ---------- @@ -440,72 +852,138 @@ def residual_unit_v3_x(data, num_filter, stride, dim_match, name, bottle_neck, * act_type = kwargs.get('version_act', 'prelu') num_group = 32 # print('in unit3') - bn1 = mx.sym.BatchNorm(data=data, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn1') - conv1 = Conv(data=bn1, num_group=num_group, num_filter=int(num_filter * 0.5), kernel=(1, 1), stride=(1, 1), + bn1 = mx.sym.BatchNorm(data=data, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn1') + conv1 = Conv(data=bn1, + num_group=num_group, + num_filter=int(num_filter * 0.5), + kernel=(1, 1), + stride=(1, 1), pad=(0, 0), - no_bias=True, workspace=workspace, name=name + '_conv1') - bn2 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn2') + no_bias=True, + workspace=workspace, + name=name + '_conv1') + bn2 = mx.sym.BatchNorm(data=conv1, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn2') act1 = Act(data=bn2, act_type=act_type, name=name + '_relu1') - conv2 = Conv(data=act1, num_group=num_group, num_filter=int(num_filter * 0.5), kernel=(3, 3), stride=(1, 1), + conv2 = Conv(data=act1, + num_group=num_group, + num_filter=int(num_filter * 0.5), + kernel=(3, 3), + stride=(1, 1), pad=(1, 1), - no_bias=True, workspace=workspace, name=name + '_conv2') - bn3 = mx.sym.BatchNorm(data=conv2, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn3') + no_bias=True, + workspace=workspace, + name=name + '_conv2') + bn3 = mx.sym.BatchNorm(data=conv2, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn3') act2 = Act(data=bn3, act_type=act_type, name=name + '_relu2') - conv3 = Conv(data=act2, num_filter=num_filter, kernel=(1, 1), stride=stride, pad=(0, 0), no_bias=True, - workspace=workspace, name=name + '_conv3') - bn4 = mx.sym.BatchNorm(data=conv3, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn4') + conv3 = Conv(data=act2, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + pad=(0, 0), + no_bias=True, + workspace=workspace, + name=name + '_conv3') + bn4 = mx.sym.BatchNorm(data=conv3, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn4') if use_se: # se begin - body = mx.sym.Pooling(data=bn4, global_pool=True, kernel=(7, 7), pool_type='avg', name=name + '_se_pool1') - body = Conv(data=body, num_filter=num_filter // 16, kernel=(1, 1), stride=(1, 1), pad=(0, 0), - name=name + "_se_conv1", workspace=workspace) + body = mx.sym.Pooling(data=bn4, + global_pool=True, + kernel=(7, 7), + pool_type='avg', + name=name + '_se_pool1') + body = Conv(data=body, + num_filter=num_filter // 16, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv1", + workspace=workspace) body = Act(data=body, act_type=act_type, name=name + '_se_relu1') - body = Conv(data=body, num_filter=num_filter, kernel=(1, 1), stride=(1, 1), pad=(0, 0), - name=name + "_se_conv2", workspace=workspace) - body = mx.symbol.Activation(data=body, act_type='sigmoid', name=name + "_se_sigmoid") + body = Conv(data=body, + num_filter=num_filter, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv2", + workspace=workspace) + body = mx.symbol.Activation(data=body, + act_type='sigmoid', + name=name + "_se_sigmoid") bn4 = mx.symbol.broadcast_mul(bn4, body) # se end if dim_match: shortcut = data else: - conv1sc = Conv(data=data, num_filter=num_filter, kernel=(1, 1), stride=stride, no_bias=True, - workspace=workspace, name=name + '_conv1sc') - shortcut = mx.sym.BatchNorm(data=conv1sc, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_sc') + conv1sc = Conv(data=data, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + no_bias=True, + workspace=workspace, + name=name + '_conv1sc') + shortcut = mx.sym.BatchNorm(data=conv1sc, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_sc') if memonger: shortcut._set_attr(mirror_stage='True') return bn4 + shortcut -def residual_unit(data, num_filter, stride, dim_match, name, bottle_neck, **kwargs): +def residual_unit(data, num_filter, stride, dim_match, name, bottle_neck, + **kwargs): uv = kwargs.get('version_unit', 3) version_input = kwargs.get('version_input', 1) if uv == 1: if version_input == 0: - return residual_unit_v1(data, num_filter, stride, dim_match, name, bottle_neck, **kwargs) + return residual_unit_v1(data, num_filter, stride, dim_match, name, + bottle_neck, **kwargs) else: - return residual_unit_v1_L(data, num_filter, stride, dim_match, name, bottle_neck, **kwargs) + return residual_unit_v1_L(data, num_filter, stride, dim_match, + name, bottle_neck, **kwargs) elif uv == 2: - return residual_unit_v2(data, num_filter, stride, dim_match, name, bottle_neck, **kwargs) + return residual_unit_v2(data, num_filter, stride, dim_match, name, + bottle_neck, **kwargs) elif uv == 4: - return residual_unit_v4(data, num_filter, stride, dim_match, name, bottle_neck, **kwargs) + return residual_unit_v4(data, num_filter, stride, dim_match, name, + bottle_neck, **kwargs) else: - return residual_unit_v3(data, num_filter, stride, dim_match, name, bottle_neck, **kwargs) + return residual_unit_v3(data, num_filter, stride, dim_match, name, + bottle_neck, **kwargs) def resnet(units, num_stages, filter_list, num_classes, bottle_neck): bn_mom = config.bn_mom workspace = config.workspace - kwargs = {'version_se': config.net_se, - 'version_input': config.net_input, - 'version_output': config.net_output, - 'version_unit': config.net_unit, - 'version_act': config.net_act, - 'bn_mom': bn_mom, - 'workspace': workspace, - 'memonger': config.memonger, - } + kwargs = { + 'version_se': config.net_se, + 'version_input': config.net_input, + 'version_output': config.net_output, + 'version_unit': config.net_unit, + 'version_act': config.net_act, + 'bn_mom': bn_mom, + 'workspace': workspace, + 'memonger': config.memonger, + } """Return ResNet symbol of Parameters ---------- @@ -530,7 +1008,8 @@ def resnet(units, num_stages, filter_list, num_classes, bottle_neck): version_unit = kwargs.get('version_unit', 3) act_type = kwargs.get('version_act', 'prelu') memonger = kwargs.get('memonger', False) - print(version_se, version_input, version_output, version_unit, act_type, memonger) + print(version_se, version_input, version_output, version_unit, act_type, + memonger) num_unit = len(units) assert (num_unit == num_stages) data = mx.sym.Variable(name='data') @@ -543,25 +1022,59 @@ def resnet(units, num_stages, filter_list, num_classes, bottle_neck): data = mx.sym.identity(data=data, name='id') data = data - 127.5 data = data * 0.0078125 - body = Conv(data=data, num_filter=filter_list[0], kernel=(7, 7), stride=(2, 2), pad=(3, 3), - no_bias=True, name="conv0", workspace=workspace) - body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn0') + body = Conv(data=data, + num_filter=filter_list[0], + kernel=(7, 7), + stride=(2, 2), + pad=(3, 3), + no_bias=True, + name="conv0", + workspace=workspace) + body = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name='bn0') body = Act(data=body, act_type=act_type, name='relu0') # body = mx.sym.Pooling(data=body, kernel=(3, 3), stride=(2,2), pad=(1,1), pool_type='max') elif version_input == 2: - data = mx.sym.BatchNorm(data=data, fix_gamma=True, eps=2e-5, momentum=bn_mom, name='bn_data') - body = Conv(data=data, num_filter=filter_list[0], kernel=(3, 3), stride=(1, 1), pad=(1, 1), - no_bias=True, name="conv0", workspace=workspace) - body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn0') + data = mx.sym.BatchNorm(data=data, + fix_gamma=True, + eps=2e-5, + momentum=bn_mom, + name='bn_data') + body = Conv(data=data, + num_filter=filter_list[0], + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + name="conv0", + workspace=workspace) + body = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name='bn0') body = Act(data=body, act_type=act_type, name='relu0') else: data = mx.sym.identity(data=data, name='id') data = data - 127.5 data = data * 0.0078125 body = data - body = Conv(data=body, num_filter=filter_list[0], kernel=(3, 3), stride=(1, 1), pad=(1, 1), - no_bias=True, name="conv0", workspace=workspace) - body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn0') + body = Conv(data=body, + num_filter=filter_list[0], + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + name="conv0", + workspace=workspace) + body = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name='bn0') body = Act(data=body, act_type=act_type, name='relu0') for i in range(num_stages): @@ -571,18 +1084,36 @@ def resnet(units, num_stages, filter_list, num_classes, bottle_neck): # else: # body = residual_unit(body, filter_list[i+1], (2, 2), False, # name='stage%d_unit%d' % (i + 1, 1), bottle_neck=bottle_neck, **kwargs) - body = residual_unit(body, filter_list[i + 1], (2, 2), False, - name='stage%d_unit%d' % (i + 1, 1), bottle_neck=bottle_neck, **kwargs) + body = residual_unit(body, + filter_list[i + 1], (2, 2), + False, + name='stage%d_unit%d' % (i + 1, 1), + bottle_neck=bottle_neck, + **kwargs) for j in range(units[i] - 1): - body = residual_unit(body, filter_list[i + 1], (1, 1), True, name='stage%d_unit%d' % (i + 1, j + 2), - bottle_neck=bottle_neck, **kwargs) + body = residual_unit(body, + filter_list[i + 1], (1, 1), + True, + name='stage%d_unit%d' % (i + 1, j + 2), + bottle_neck=bottle_neck, + **kwargs) if config.fp16: body = mx.sym.Cast(data=body, dtype=np.float32) if bottle_neck: - body = Conv(data=body, num_filter=512, kernel=(1, 1), stride=(1, 1), pad=(0, 0), - no_bias=True, name="convd", workspace=workspace) - body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bnd') + body = Conv(data=body, + num_filter=512, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + no_bias=True, + name="convd", + workspace=workspace) + body = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name='bnd') body = Act(data=body, act_type=act_type, name='relud') fc1 = symbol_utils.get_fc1(body, num_classes, fc_type) @@ -640,7 +1171,9 @@ def get_symbol(): elif num_layers == 269: units = [3, 30, 48, 8] else: - raise ValueError("no experiments done on num_layers {}, you can do it yourself".format(num_layers)) + raise ValueError( + "no experiments done on num_layers {}, you can do it yourself". + format(num_layers)) net = resnet(units=units, num_stages=num_stages, diff --git a/recognition/partial_fc/mxnet/symbol/symbol_utils.py b/recognition/partial_fc/mxnet/symbol/symbol_utils.py index 5da2b27..73cd398 100644 --- a/recognition/partial_fc/mxnet/symbol/symbol_utils.py +++ b/recognition/partial_fc/mxnet/symbol/symbol_utils.py @@ -27,58 +27,154 @@ def Act(data, act_type, name): bn_mom = config.bn_mom -def Linear(data, num_filter=1, kernel=(1, 1), stride=(1, 1), pad=(0, 0), num_group=1, name=None, suffix=''): - conv = mx.sym.Convolution(data=data, num_filter=num_filter, kernel=kernel, num_group=num_group, stride=stride, - pad=pad, no_bias=True, name='%s%s_conv2d' % (name, suffix)) - bn = mx.sym.BatchNorm(data=conv, name='%s%s_batchnorm' % (name, suffix), fix_gamma=False, momentum=bn_mom) +def Linear(data, + num_filter=1, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + num_group=1, + name=None, + suffix=''): + conv = mx.sym.Convolution(data=data, + num_filter=num_filter, + kernel=kernel, + num_group=num_group, + stride=stride, + pad=pad, + no_bias=True, + name='%s%s_conv2d' % (name, suffix)) + bn = mx.sym.BatchNorm(data=conv, + name='%s%s_batchnorm' % (name, suffix), + fix_gamma=False, + momentum=bn_mom) return bn def get_fc1(last_conv, num_classes, fc_type, input_channel=512): body = last_conv if fc_type == 'Z': - body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn1') + body = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name='bn1') body = mx.symbol.Dropout(data=body, p=0.4) fc1 = body elif fc_type == 'E': - body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn1') + body = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name='bn1') body = mx.symbol.Dropout(data=body, p=0.4) - fc1 = mx.sym.FullyConnected(data=body, num_hidden=num_classes, name='pre_fc1') - fc1 = mx.sym.BatchNorm(data=fc1, fix_gamma=True, eps=2e-5, momentum=bn_mom, name='fc1') + fc1 = mx.sym.FullyConnected(data=body, + num_hidden=num_classes, + name='pre_fc1') + fc1 = mx.sym.BatchNorm(data=fc1, + fix_gamma=True, + eps=2e-5, + momentum=bn_mom, + name='fc1') elif fc_type == 'FC': - body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn1') - fc1 = mx.sym.FullyConnected(data=body, num_hidden=num_classes, name='pre_fc1') - fc1 = mx.sym.BatchNorm(data=fc1, fix_gamma=True, eps=2e-5, momentum=bn_mom, name='fc1') + body = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name='bn1') + fc1 = mx.sym.FullyConnected(data=body, + num_hidden=num_classes, + name='pre_fc1') + fc1 = mx.sym.BatchNorm(data=fc1, + fix_gamma=True, + eps=2e-5, + momentum=bn_mom, + name='fc1') elif fc_type == 'SFC': - body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn1') - body = Conv(data=body, num_filter=input_channel, kernel=(3, 3), stride=(2, 2), pad=(1, 1), - no_bias=True, name="convf", num_group=input_channel) - body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bnf') + body = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name='bn1') + body = Conv(data=body, + num_filter=input_channel, + kernel=(3, 3), + stride=(2, 2), + pad=(1, 1), + no_bias=True, + name="convf", + num_group=input_channel) + body = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name='bnf') body = Act(data=body, act_type=config.net_act, name='reluf') - body = Conv(data=body, num_filter=input_channel, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="convf2") - body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bnf2') + body = Conv(data=body, + num_filter=input_channel, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + name="convf2") + body = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name='bnf2') body = Act(data=body, act_type=config.net_act, name='reluf2') - fc1 = mx.sym.FullyConnected(data=body, num_hidden=num_classes, name='pre_fc1') - fc1 = mx.sym.BatchNorm(data=fc1, fix_gamma=True, eps=2e-5, momentum=bn_mom, name='fc1') + fc1 = mx.sym.FullyConnected(data=body, + num_hidden=num_classes, + name='pre_fc1') + fc1 = mx.sym.BatchNorm(data=fc1, + fix_gamma=True, + eps=2e-5, + momentum=bn_mom, + name='fc1') elif fc_type == 'GAP': - bn1 = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn1') + bn1 = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name='bn1') relu1 = Act(data=bn1, act_type=config.net_act, name='relu1') # Although kernel is not used here when global_pool=True, we should put one - pool1 = mx.sym.Pooling(data=relu1, global_pool=True, kernel=(7, 7), pool_type='avg', name='pool1') + pool1 = mx.sym.Pooling(data=relu1, + global_pool=True, + kernel=(7, 7), + pool_type='avg', + name='pool1') flat = mx.sym.Flatten(data=pool1) - fc1 = mx.sym.FullyConnected(data=flat, num_hidden=num_classes, name='pre_fc1') - fc1 = mx.sym.BatchNorm(data=fc1, fix_gamma=True, eps=2e-5, momentum=bn_mom, name='fc1') + fc1 = mx.sym.FullyConnected(data=flat, + num_hidden=num_classes, + name='pre_fc1') + fc1 = mx.sym.BatchNorm(data=fc1, + fix_gamma=True, + eps=2e-5, + momentum=bn_mom, + name='fc1') elif fc_type == 'GNAP': # mobilefacenet++ filters_in = 512 # param in mobilefacenet if num_classes > filters_in: - body = mx.sym.Convolution(data=last_conv, num_filter=num_classes, kernel=(1, 1), stride=(1, 1), pad=(0, 0), - no_bias=True, name='convx') - body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=0.9, name='convx_bn') + body = mx.sym.Convolution(data=last_conv, + num_filter=num_classes, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + no_bias=True, + name='convx') + body = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=0.9, + name='convx_bn') body = Act(data=body, act_type=config.net_act, name='convx_relu') filters_in = num_classes else: body = last_conv - body = mx.sym.BatchNorm(data=body, fix_gamma=True, eps=2e-5, momentum=0.9, name='bn6f') + body = mx.sym.BatchNorm(data=body, + fix_gamma=True, + eps=2e-5, + momentum=0.9, + name='bn6f') spatial_norm = body * body spatial_norm = mx.sym.sum(data=spatial_norm, axis=1, keepdims=True) @@ -87,38 +183,93 @@ def get_fc1(last_conv, num_classes, fc_type, input_channel=512): spatial_mean = mx.sym.mean(spatial_sqrt) spatial_div_inverse = mx.sym.broadcast_div(spatial_mean, spatial_sqrt) - spatial_attention_inverse = mx.symbol.tile(spatial_div_inverse, reps=(1, filters_in, 1, 1)) + spatial_attention_inverse = mx.symbol.tile(spatial_div_inverse, + reps=(1, filters_in, 1, 1)) body = body * spatial_attention_inverse # body = mx.sym.broadcast_mul(body, spatial_div_inverse) - fc1 = mx.sym.Pooling(body, kernel=(7, 7), global_pool=True, pool_type='avg') + fc1 = mx.sym.Pooling(body, + kernel=(7, 7), + global_pool=True, + pool_type='avg') if num_classes < filters_in: - fc1 = mx.sym.BatchNorm(data=fc1, fix_gamma=True, eps=2e-5, momentum=0.9, name='bn6w') - fc1 = mx.sym.FullyConnected(data=fc1, num_hidden=num_classes, name='pre_fc1') + fc1 = mx.sym.BatchNorm(data=fc1, + fix_gamma=True, + eps=2e-5, + momentum=0.9, + name='bn6w') + fc1 = mx.sym.FullyConnected(data=fc1, + num_hidden=num_classes, + name='pre_fc1') else: fc1 = mx.sym.Flatten(data=fc1) - fc1 = mx.sym.BatchNorm(data=fc1, fix_gamma=True, eps=2e-5, momentum=0.9, name='fc1') + fc1 = mx.sym.BatchNorm(data=fc1, + fix_gamma=True, + eps=2e-5, + momentum=0.9, + name='fc1') elif fc_type == "GDC": # mobilefacenet_v1 - conv_6_dw = Linear(last_conv, num_filter=input_channel, num_group=input_channel, kernel=(7, 7), pad=(0, 0), - stride=(1, 1), name="conv_6dw7_7") - conv_6_f = mx.sym.FullyConnected(data=conv_6_dw, num_hidden=num_classes, name='pre_fc1') - fc1 = mx.sym.BatchNorm(data=conv_6_f, fix_gamma=True, eps=2e-5, momentum=bn_mom, name='fc1') + conv_6_dw = Linear(last_conv, + num_filter=input_channel, + num_group=input_channel, + kernel=(7, 7), + pad=(0, 0), + stride=(1, 1), + name="conv_6dw7_7") + conv_6_f = mx.sym.FullyConnected(data=conv_6_dw, + num_hidden=num_classes, + name='pre_fc1') + fc1 = mx.sym.BatchNorm(data=conv_6_f, + fix_gamma=True, + eps=2e-5, + momentum=bn_mom, + name='fc1') elif fc_type == 'F': - body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn1') + body = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name='bn1') body = mx.symbol.Dropout(data=body, p=0.4) - fc1 = mx.sym.FullyConnected(data=body, num_hidden=num_classes, name='fc1') + fc1 = mx.sym.FullyConnected(data=body, + num_hidden=num_classes, + name='fc1') elif fc_type == 'G': - body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn1') - fc1 = mx.sym.FullyConnected(data=body, num_hidden=num_classes, name='fc1') + body = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name='bn1') + fc1 = mx.sym.FullyConnected(data=body, + num_hidden=num_classes, + name='fc1') elif fc_type == 'H': - fc1 = mx.sym.FullyConnected(data=body, num_hidden=num_classes, name='fc1') + fc1 = mx.sym.FullyConnected(data=body, + num_hidden=num_classes, + name='fc1') elif fc_type == 'I': - body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn1') - fc1 = mx.sym.FullyConnected(data=body, num_hidden=num_classes, name='pre_fc1') - fc1 = mx.sym.BatchNorm(data=fc1, fix_gamma=True, eps=2e-5, momentum=bn_mom, name='fc1') + body = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name='bn1') + fc1 = mx.sym.FullyConnected(data=body, + num_hidden=num_classes, + name='pre_fc1') + fc1 = mx.sym.BatchNorm(data=fc1, + fix_gamma=True, + eps=2e-5, + momentum=bn_mom, + name='fc1') elif fc_type == 'J': - fc1 = mx.sym.FullyConnected(data=body, num_hidden=num_classes, name='pre_fc1') - fc1 = mx.sym.BatchNorm(data=fc1, fix_gamma=True, eps=2e-5, momentum=bn_mom, name='fc1') + fc1 = mx.sym.FullyConnected(data=body, + num_hidden=num_classes, + name='pre_fc1') + fc1 = mx.sym.BatchNorm(data=fc1, + fix_gamma=True, + eps=2e-5, + momentum=bn_mom, + name='fc1') return fc1 @@ -145,21 +296,54 @@ def residual_unit_v3(data, num_filter, stride, dim_match, name, **kwargs): workspace = kwargs.get('workspace', 256) memonger = kwargs.get('memonger', False) # print('in unit3') - bn1 = mx.sym.BatchNorm(data=data, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn1') - conv1 = Conv(data=bn1, num_filter=num_filter, kernel=(3, 3), stride=(1, 1), pad=(1, 1), - no_bias=True, workspace=workspace, name=name + '_conv1') - bn2 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn2') + bn1 = mx.sym.BatchNorm(data=data, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn1') + conv1 = Conv(data=bn1, + num_filter=num_filter, + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv1') + bn2 = mx.sym.BatchNorm(data=conv1, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn2') act1 = Act(data=bn2, act_type=config.net_act, name=name + '_relu1') - conv2 = Conv(data=act1, num_filter=num_filter, kernel=(3, 3), stride=stride, pad=(1, 1), - no_bias=True, workspace=workspace, name=name + '_conv2') - bn3 = mx.sym.BatchNorm(data=conv2, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn3') + conv2 = Conv(data=act1, + num_filter=num_filter, + kernel=(3, 3), + stride=stride, + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv2') + bn3 = mx.sym.BatchNorm(data=conv2, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn3') if dim_match: shortcut = data else: - conv1sc = Conv(data=data, num_filter=num_filter, kernel=(1, 1), stride=stride, no_bias=True, - workspace=workspace, name=name + '_conv1sc') - shortcut = mx.sym.BatchNorm(data=conv1sc, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_sc') + conv1sc = Conv(data=data, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + no_bias=True, + workspace=workspace, + name=name + '_conv1sc') + shortcut = mx.sym.BatchNorm(data=conv1sc, + fix_gamma=False, + momentum=bn_mom, + eps=2e-5, + name=name + '_sc') if memonger: shortcut._set_attr(mirror_stage='True') return bn3 + shortcut @@ -191,68 +375,172 @@ def residual_unit_v1l(data, num_filter, stride, dim_match, name, bottle_neck): act_type = config.net_act # print('in unit1') if bottle_neck: - conv1 = Conv(data=data, num_filter=int(num_filter * 0.25), kernel=(1, 1), stride=(1, 1), pad=(0, 0), - no_bias=True, workspace=workspace, name=name + '_conv1') - bn1 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn1') + conv1 = Conv(data=data, + num_filter=int(num_filter * 0.25), + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + no_bias=True, + workspace=workspace, + name=name + '_conv1') + bn1 = mx.sym.BatchNorm(data=conv1, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn1') act1 = Act(data=bn1, act_type=act_type, name=name + '_relu1') - conv2 = Conv(data=act1, num_filter=int(num_filter * 0.25), kernel=(3, 3), stride=(1, 1), pad=(1, 1), - no_bias=True, workspace=workspace, name=name + '_conv2') - bn2 = mx.sym.BatchNorm(data=conv2, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn2') + conv2 = Conv(data=act1, + num_filter=int(num_filter * 0.25), + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv2') + bn2 = mx.sym.BatchNorm(data=conv2, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn2') act2 = Act(data=bn2, act_type=act_type, name=name + '_relu2') - conv3 = Conv(data=act2, num_filter=num_filter, kernel=(1, 1), stride=stride, pad=(0, 0), no_bias=True, - workspace=workspace, name=name + '_conv3') - bn3 = mx.sym.BatchNorm(data=conv3, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn3') + conv3 = Conv(data=act2, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + pad=(0, 0), + no_bias=True, + workspace=workspace, + name=name + '_conv3') + bn3 = mx.sym.BatchNorm(data=conv3, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn3') if use_se: # se begin - body = mx.sym.Pooling(data=bn3, global_pool=True, kernel=(7, 7), pool_type='avg', name=name + '_se_pool1') - body = Conv(data=body, num_filter=num_filter // 16, kernel=(1, 1), stride=(1, 1), pad=(0, 0), - name=name + "_se_conv1", workspace=workspace) + body = mx.sym.Pooling(data=bn3, + global_pool=True, + kernel=(7, 7), + pool_type='avg', + name=name + '_se_pool1') + body = Conv(data=body, + num_filter=num_filter // 16, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv1", + workspace=workspace) body = Act(data=body, act_type=act_type, name=name + '_se_relu1') - body = Conv(data=body, num_filter=num_filter, kernel=(1, 1), stride=(1, 1), pad=(0, 0), - name=name + "_se_conv2", workspace=workspace) - body = mx.symbol.Activation(data=body, act_type='sigmoid', name=name + "_se_sigmoid") + body = Conv(data=body, + num_filter=num_filter, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv2", + workspace=workspace) + body = mx.symbol.Activation(data=body, + act_type='sigmoid', + name=name + "_se_sigmoid") bn3 = mx.symbol.broadcast_mul(bn3, body) # se end if dim_match: shortcut = data else: - conv1sc = Conv(data=data, num_filter=num_filter, kernel=(1, 1), stride=stride, no_bias=True, - workspace=workspace, name=name + '_conv1sc') - shortcut = mx.sym.BatchNorm(data=conv1sc, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_sc') + conv1sc = Conv(data=data, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + no_bias=True, + workspace=workspace, + name=name + '_conv1sc') + shortcut = mx.sym.BatchNorm(data=conv1sc, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_sc') if memonger: shortcut._set_attr(mirror_stage='True') - return Act(data=bn3 + shortcut, act_type=act_type, name=name + '_relu3') + return Act(data=bn3 + shortcut, + act_type=act_type, + name=name + '_relu3') else: - conv1 = Conv(data=data, num_filter=num_filter, kernel=(3, 3), stride=(1, 1), pad=(1, 1), - no_bias=True, workspace=workspace, name=name + '_conv1') - bn1 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_bn1') + conv1 = Conv(data=data, + num_filter=num_filter, + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv1') + bn1 = mx.sym.BatchNorm(data=conv1, + fix_gamma=False, + momentum=bn_mom, + eps=2e-5, + name=name + '_bn1') act1 = Act(data=bn1, act_type=act_type, name=name + '_relu1') - conv2 = Conv(data=act1, num_filter=num_filter, kernel=(3, 3), stride=stride, pad=(1, 1), - no_bias=True, workspace=workspace, name=name + '_conv2') - bn2 = mx.sym.BatchNorm(data=conv2, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_bn2') + conv2 = Conv(data=act1, + num_filter=num_filter, + kernel=(3, 3), + stride=stride, + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv2') + bn2 = mx.sym.BatchNorm(data=conv2, + fix_gamma=False, + momentum=bn_mom, + eps=2e-5, + name=name + '_bn2') if use_se: # se begin - body = mx.sym.Pooling(data=bn2, global_pool=True, kernel=(7, 7), pool_type='avg', name=name + '_se_pool1') - body = Conv(data=body, num_filter=num_filter // 16, kernel=(1, 1), stride=(1, 1), pad=(0, 0), - name=name + "_se_conv1", workspace=workspace) + body = mx.sym.Pooling(data=bn2, + global_pool=True, + kernel=(7, 7), + pool_type='avg', + name=name + '_se_pool1') + body = Conv(data=body, + num_filter=num_filter // 16, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv1", + workspace=workspace) body = Act(data=body, act_type=act_type, name=name + '_se_relu1') - body = Conv(data=body, num_filter=num_filter, kernel=(1, 1), stride=(1, 1), pad=(0, 0), - name=name + "_se_conv2", workspace=workspace) - body = mx.symbol.Activation(data=body, act_type='sigmoid', name=name + "_se_sigmoid") + body = Conv(data=body, + num_filter=num_filter, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv2", + workspace=workspace) + body = mx.symbol.Activation(data=body, + act_type='sigmoid', + name=name + "_se_sigmoid") bn2 = mx.symbol.broadcast_mul(bn2, body) # se end if dim_match: shortcut = data else: - conv1sc = Conv(data=data, num_filter=num_filter, kernel=(1, 1), stride=stride, no_bias=True, - workspace=workspace, name=name + '_conv1sc') - shortcut = mx.sym.BatchNorm(data=conv1sc, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_sc') + conv1sc = Conv(data=data, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + no_bias=True, + workspace=workspace, + name=name + '_conv1sc') + shortcut = mx.sym.BatchNorm(data=conv1sc, + fix_gamma=False, + momentum=bn_mom, + eps=2e-5, + name=name + '_sc') if memonger: shortcut._set_attr(mirror_stage='True') - return Act(data=bn2 + shortcut, act_type=act_type, name=name + '_relu3') + return Act(data=bn2 + shortcut, + act_type=act_type, + name=name + '_relu3') def get_head(data, version_input, num_filter): @@ -263,18 +551,46 @@ def get_head(data, version_input, num_filter): data = data * 0.0078125 # data = mx.sym.BatchNorm(data=data, fix_gamma=True, eps=2e-5, momentum=bn_mom, name='bn_data') if version_input == 0: - body = Conv(data=data, num_filter=num_filter, kernel=(7, 7), stride=(2, 2), pad=(3, 3), - no_bias=True, name="conv0", workspace=workspace) - body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn0') + body = Conv(data=data, + num_filter=num_filter, + kernel=(7, 7), + stride=(2, 2), + pad=(3, 3), + no_bias=True, + name="conv0", + workspace=workspace) + body = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name='bn0') body = Act(data=body, act_type=config.net_act, name='relu0') - body = mx.sym.Pooling(data=body, kernel=(3, 3), stride=(2, 2), pad=(1, 1), pool_type='max') + body = mx.sym.Pooling(data=body, + kernel=(3, 3), + stride=(2, 2), + pad=(1, 1), + pool_type='max') else: body = data _num_filter = min(num_filter, 64) - body = Conv(data=body, num_filter=_num_filter, kernel=(3, 3), stride=(1, 1), pad=(1, 1), - no_bias=True, name="conv0", workspace=workspace) - body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn0') + body = Conv(data=body, + num_filter=_num_filter, + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + name="conv0", + workspace=workspace) + body = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name='bn0') body = Act(data=body, act_type=config.net_act, name='relu0') # body = residual_unit_v3(body, _num_filter, (2, 2), False, name='head', **kwargs) - body = residual_unit_v1l(body, _num_filter, (2, 2), False, name='head', bottle_neck=False) + body = residual_unit_v1l(body, + _num_filter, (2, 2), + False, + name='head', + bottle_neck=False) return body diff --git a/recognition/partial_fc/mxnet/train_memory.py b/recognition/partial_fc/mxnet/train_memory.py index 1decd25..f6bf626 100644 --- a/recognition/partial_fc/mxnet/train_memory.py +++ b/recognition/partial_fc/mxnet/train_memory.py @@ -38,14 +38,18 @@ def parse_args(): args, rest = parser.parse_known_args() default.generate_config(args.loss, args.dataset, args.network) - parser.add_argument('--models-root', default="./test", help='root directory to save model.') + parser.add_argument('--models-root', + default="./test", + help='root directory to save model.') args = parser.parse_args() return args def set_logger(logger, rank, models_root): - formatter = logging.Formatter("rank-id:" + str(rank) + ":%(asctime)s-%(message)s") - file_handler = logging.FileHandler(os.path.join(models_root, "%d_hist.log" % rank)) + formatter = logging.Formatter("rank-id:" + str(rank) + + ":%(asctime)s-%(message)s") + file_handler = logging.FileHandler( + os.path.join(models_root, "%d_hist.log" % rank)) stream_handler = logging.StreamHandler(sys.stdout) file_handler.setFormatter(formatter) stream_handler.setFormatter(formatter) @@ -89,17 +93,24 @@ def train_net(): # We equally store the class centers (softmax linear transformation matrix) on all GPUs in order. num_local = (config.num_classes + size - 1) // size num_sample = int(num_local * config.sample_ratio) - memory_bank = MemoryBank( - num_sample=num_sample, num_local=num_local, rank=rank, - local_rank=local_rank, embedding_size=config.embedding_size, - prefix=prefix_dir, gpu=True) + memory_bank = MemoryBank(num_sample=num_sample, + num_local=num_local, + rank=rank, + local_rank=local_rank, + embedding_size=config.embedding_size, + prefix=prefix_dir, + gpu=True) if config.debug: train_iter = DummyIter(config.batch_size, data_shape, 1000 * 10000) else: - train_iter = FaceImageIter( - batch_size=config.batch_size, data_shape=data_shape, path_imgrec=config.rec, - shuffle=True, rand_mirror=True, context=rank, context_num=size) + train_iter = FaceImageIter(batch_size=config.batch_size, + data_shape=data_shape, + path_imgrec=config.rec, + shuffle=True, + rand_mirror=True, + context=rank, + context_num=size) train_data_iter = mx.io.PrefetchingIter(train_iter) esym, save_symbol = get_symbol_embedding() @@ -132,9 +143,9 @@ def train_net(): memory_optimizer=memory_bank_optimizer) # if not config.debug and local_rank == 0: - cb_vert = CallBackVertification(esym, train_module) - cb_speed = CallBackLogging(rank, size, prefix_dir) - cb_save = CallBackModelSave(save_symbol, train_module, prefix, rank) + cb_vert = CallBackVertification(esym, train_module) + cb_speed = CallBackLogging(rank, size, prefix_dir) + cb_save = CallBackModelSave(save_symbol, train_module, prefix, rank) cb_center_save = CallBackCenterSave(memory_bank) def call_back_fn(params): @@ -144,11 +155,10 @@ def train_net(): cb_center_save(params) cb_save(params) - train_module.fit( - train_data_iter, - optimizer_params=backbone_kwargs, - initializer=mx.init.Normal(0.1), - batch_end_callback=call_back_fn) + train_module.fit(train_data_iter, + optimizer_params=backbone_kwargs, + initializer=mx.init.Normal(0.1), + batch_end_callback=call_back_fn) if __name__ == '__main__': diff --git a/recognition/partial_fc/pytorch/IJB/IJB_11.py b/recognition/partial_fc/pytorch/IJB/IJB_11.py index 7027a9d..48bf432 100644 --- a/recognition/partial_fc/pytorch/IJB/IJB_11.py +++ b/recognition/partial_fc/pytorch/IJB/IJB_11.py @@ -40,7 +40,10 @@ parser.add_argument('--result-dir', default='.', type=str, help='') parser.add_argument('--gpu', default=7, type=int, help='gpu id') parser.add_argument('--batch-size', default=32, type=int, help='') parser.add_argument('--job', default='insightface', type=str, help='job name') -parser.add_argument('--target', default='IJBC', type=str, help='target, set to IJBC or IJBB') +parser.add_argument('--target', + default='IJBC', + type=str, + help='target, set to IJBC or IJBB') args = parser.parse_args() target = args.target @@ -58,6 +61,7 @@ job = args.job # hvd.init() # rank_size = hvd.size() + # 将一个list尽量均分成n份，限制len(list)==n，份数大于原list内元素个数则分配空list[] def divideIntoNstrand(listTemp, n): twoList = [[] for i in range(n)] @@ -99,6 +103,7 @@ def read_image_feature(path): # In[ ]: + # # def get_image_feature(img_path, files_list, model_path, epoch, gpu_id): # batch_size = 125 @@ -145,14 +150,15 @@ def get_image_feature(img_path, img_list_path, model_path, epoch, gpu_id): faceness_scores = [] img_feats = [] for img_index, each_line in enumerate(files): - if img_index%500==0: - print('processing', img_index) + if img_index % 500 == 0: + print('processing', img_index) name_lmk_score = each_line.strip().split(' ') img_name = os.path.join(img_path, name_lmk_score[0]) img = cv2.imread(img_name) - lmk = np.array([float(x) for x in name_lmk_score[1:-1]], dtype=np.float32) - lmk = lmk.reshape( (5,2) ) - img_feats.append(embedding.get(img,lmk)) + lmk = np.array([float(x) for x in name_lmk_score[1:-1]], + dtype=np.float32) + lmk = lmk.reshape((5, 2)) + img_feats.append(embedding.get(img, lmk)) faceness_scores.append(name_lmk_score[-1]) img_feats = np.array(img_feats).astype(np.float32) faceness_scores = np.array(faceness_scores).astype(np.float32) @@ -161,6 +167,7 @@ def get_image_feature(img_path, img_list_path, model_path, epoch, gpu_id): #faceness_scores = np.ones( (len(files), ), dtype=np.float32 ) return img_feats, faceness_scores + # In[ ]: @@ -175,22 +182,26 @@ def image2template_feature(img_feats=None, templates=None, medias=None): for count_template, uqt in enumerate(unique_templates): - (ind_t,) = np.where(templates == uqt) + (ind_t, ) = np.where(templates == uqt) face_norm_feats = img_feats[ind_t] face_medias = medias[ind_t] - unique_medias, unique_media_counts = np.unique(face_medias, return_counts=True) + unique_medias, unique_media_counts = np.unique(face_medias, + return_counts=True) media_norm_feats = [] for u, ct in zip(unique_medias, unique_media_counts): - (ind_m,) = np.where(face_medias == u) + (ind_m, ) = np.where(face_medias == u) if ct == 1: media_norm_feats += [face_norm_feats[ind_m]] else: # image features from the same video will be aggregated into one feature - media_norm_feats += [np.mean(face_norm_feats[ind_m], axis=0, keepdims=True)] + media_norm_feats += [ + np.mean(face_norm_feats[ind_m], axis=0, keepdims=True) + ] media_norm_feats = np.array(media_norm_feats) # media_norm_feats = media_norm_feats / np.sqrt(np.sum(media_norm_feats ** 2, -1, keepdims=True)) template_feats[count_template] = np.sum(media_norm_feats, axis=0) if count_template % 2000 == 0: - print('Finish Calculating {} template features.'.format(count_template)) + print('Finish Calculating {} template features.'.format( + count_template)) # template_norm_feats = template_feats / np.sqrt(np.sum(template_feats ** 2, -1, keepdims=True)) template_norm_feats = sklearn.preprocessing.normalize(template_feats) # print(template_norm_feats.shape) @@ -200,7 +211,10 @@ def image2template_feature(img_feats=None, templates=None, medias=None): # In[ ]: -def verification(template_norm_feats=None, unique_templates=None, p1=None, p2=None): +def verification(template_norm_feats=None, + unique_templates=None, + p1=None, + p2=None): # ========================================================== # Compute set-to-set Similarity Score. # ========================================================== @@ -208,11 +222,13 @@ def verification(template_norm_feats=None, unique_templates=None, p1=None, p2=No for count_template, uqt in enumerate(unique_templates): template2id[uqt] = count_template - score = np.zeros((len(p1),)) # save cosine distance between pairs + score = np.zeros((len(p1), )) # save cosine distance between pairs total_pairs = np.array(range(len(p1))) batchsize = 100000 # small batchsize instead of all pairs in one batch due to the memory limiation - sublists = [total_pairs[i:i + batchsize] for i in range(0, len(p1), batchsize)] + sublists = [ + total_pairs[i:i + batchsize] for i in range(0, len(p1), batchsize) + ] total_sublists = len(sublists) for c, s in enumerate(sublists): feat1 = template_norm_feats[template2id[p1[s]]] @@ -225,14 +241,19 @@ def verification(template_norm_feats=None, unique_templates=None, p1=None, p2=No # In[ ]: -def verification2(template_norm_feats=None, unique_templates=None, p1=None, p2=None): +def verification2(template_norm_feats=None, + unique_templates=None, + p1=None, + p2=None): template2id = np.zeros((max(unique_templates) + 1, 1), dtype=int) for count_template, uqt in enumerate(unique_templates): template2id[uqt] = count_template - score = np.zeros((len(p1),)) # save cosine distance between pairs + score = np.zeros((len(p1), )) # save cosine distance between pairs total_pairs = np.array(range(len(p1))) batchsize = 100000 # small batchsize instead of all pairs in one batch due to the memory limiation - sublists = [total_pairs[i:i + batchsize] for i in range(0, len(p1), batchsize)] + sublists = [ + total_pairs[i:i + batchsize] for i in range(0, len(p1), batchsize) + ] total_sublists = len(sublists) for c, s in enumerate(sublists): feat1 = template_norm_feats[template2id[p1[s]]] @@ -264,13 +285,13 @@ assert target == 'IJBC' or target == 'IJBB' # ============================================================= start = timeit.default_timer() templates, medias = read_template_media_list( - os.path.join('%s/meta' % image_path, '%s_face_tid_mid.txt' % target.lower())) + os.path.join('%s/meta' % image_path, + '%s_face_tid_mid.txt' % target.lower())) stop = timeit.default_timer() print('Time: %.2f s. ' % (stop - start)) # In[ ]: - # ============================================================= # load template pairs for template-to-template verification # tid : template id, label : 1/0 @@ -279,7 +300,8 @@ print('Time: %.2f s. ' % (stop - start)) # ============================================================= start = timeit.default_timer() p1, p2, label = read_template_pair_list( - os.path.join('%s/meta' % image_path, '%s_template_pair_label.txt' % target.lower())) + os.path.join('%s/meta' % image_path, + '%s_template_pair_label.txt' % target.lower())) stop = timeit.default_timer() print('Time: %.2f s. ' % (stop - start)) @@ -287,7 +309,6 @@ print('Time: %.2f s. ' % (stop - start)) # In[ ]: - # ============================================================= # load image features # format: @@ -305,22 +326,21 @@ print('Time: %.2f s. ' % (stop - start)) # for i in range(rank_size): # img_feats, faceness_scores = get_image_feature(img_path, files_list, model_path, epoch, gpu_id) - start = timeit.default_timer() img_path = '%s/loose_crop' % image_path img_list_path = '%s/meta/%s_name_5pts_score.txt' % (image_path, target.lower()) -img_feats, faceness_scores = get_image_feature(img_path, img_list_path, model_path, epoch, gpu_id) - +img_feats, faceness_scores = get_image_feature(img_path, img_list_path, + model_path, epoch, gpu_id) stop = timeit.default_timer() print('Time: %.2f s. ' % (stop - start)) -print('Feature Shape: ({} , {}) .'.format(img_feats.shape[0], img_feats.shape[1])) +print('Feature Shape: ({} , {}) .'.format(img_feats.shape[0], + img_feats.shape[1])) # # Step3: Get Template Features # In[ ]: - # ============================================================= # compute template features from image features. # ============================================================= @@ -332,12 +352,12 @@ start = timeit.default_timer() # 1. FaceScore （Feature Norm） # 2. FaceScore （Detector） - if use_flip_test: # concat --- F1 # img_input_feats = img_feats # add --- F2 - img_input_feats = img_feats[:, 0:img_feats.shape[1] // 2] + img_feats[:, img_feats.shape[1] // 2:] + img_input_feats = img_feats[:, 0:img_feats.shape[1] // + 2] + img_feats[:, img_feats.shape[1] // 2:] else: img_input_feats = img_feats[:, 0:img_feats.shape[1] // 2] @@ -345,7 +365,8 @@ if use_norm_score: img_input_feats = img_input_feats else: # normalise features to remove norm information - img_input_feats = img_input_feats / np.sqrt(np.sum(img_input_feats ** 2, -1, keepdims=True)) + img_input_feats = img_input_feats / np.sqrt( + np.sum(img_input_feats**2, -1, keepdims=True)) if use_detector_score: print(img_input_feats.shape, faceness_scores.shape) @@ -354,7 +375,8 @@ if use_detector_score: else: img_input_feats = img_input_feats -template_norm_feats, unique_templates = image2template_feature(img_input_feats, templates, medias) +template_norm_feats, unique_templates = image2template_feature( + img_input_feats, templates, medias) stop = timeit.default_timer() print('Time: %.2f s. ' % (stop - start)) @@ -362,7 +384,6 @@ print('Time: %.2f s. ' % (stop - start)) # In[ ]: - # ============================================================= # compute verification scores between template pairs. # ============================================================= @@ -373,8 +394,7 @@ print('Time: %.2f s. ' % (stop - start)) # In[ ]: -save_path = result_dir+'/%s_result' % target - +save_path = result_dir + '/%s_result' % target if not os.path.exists(save_path): os.makedirs(save_path) @@ -386,7 +406,6 @@ np.save(score_save_file, score) # In[ ]: - files = [score_save_file] methods = [] scores = [] @@ -396,9 +415,10 @@ for file in files: methods = np.array(methods) scores = dict(zip(methods, scores)) -colours = dict(zip(methods, sample_colours_from_colourmap(methods.shape[0], 'Set2'))) +colours = dict( + zip(methods, sample_colours_from_colourmap(methods.shape[0], 'Set2'))) # x_labels = [1/(10**x) for x in np.linspace(6, 0, 6)] -x_labels = [10 ** -6, 10 ** -5, 10 ** -4, 10 ** -3, 10 ** -2, 10 ** -1] +x_labels = [10**-6, 10**-5, 10**-4, 10**-3, 10**-2, 10**-1] tpr_fpr_table = PrettyTable(['Methods'] + [str(x) for x in x_labels]) fig = plt.figure() for method in methods: @@ -406,16 +426,21 @@ for method in methods: roc_auc = auc(fpr, tpr) fpr = np.flipud(fpr) tpr = np.flipud(tpr) # select largest tpr at same fpr - plt.plot(fpr, tpr, color=colours[method], lw=1, - label=('[%s (AUC = %0.4f %%)]' % (method.split('-')[-1], roc_auc * 100))) + plt.plot(fpr, + tpr, + color=colours[method], + lw=1, + label=('[%s (AUC = %0.4f %%)]' % + (method.split('-')[-1], roc_auc * 100))) tpr_fpr_row = [] tpr_fpr_row.append("%s-%s" % (method, target)) for fpr_iter in np.arange(len(x_labels)): - _, min_index = min(list(zip(abs(fpr - x_labels[fpr_iter]), range(len(fpr))))) + _, min_index = min( + list(zip(abs(fpr - x_labels[fpr_iter]), range(len(fpr))))) # tpr_fpr_row.append('%.4f' % tpr[min_index]) tpr_fpr_row.append('%.2f' % (tpr[min_index] * 100)) tpr_fpr_table.add_row(tpr_fpr_row) -plt.xlim([10 ** -6, 0.1]) +plt.xlim([10**-6, 0.1]) plt.ylim([0.3, 1.0]) plt.grid(linestyle='--', linewidth=1) plt.xticks(x_labels) diff --git a/recognition/partial_fc/pytorch/IJB/IJB_11_Batch.py b/recognition/partial_fc/pytorch/IJB/IJB_11_Batch.py index 212d1c6..8a2fb0d 100644 --- a/recognition/partial_fc/pytorch/IJB/IJB_11_Batch.py +++ b/recognition/partial_fc/pytorch/IJB/IJB_11_Batch.py @@ -39,7 +39,10 @@ parser.add_argument('--result-dir', default='.', type=str, help='') parser.add_argument('--gpu', default=7, type=int, help='gpu id') parser.add_argument('--batch-size', default=128, type=int, help='') parser.add_argument('--job', default='insightface', type=str, help='job name') -parser.add_argument('--target', default='IJBC', type=str, help='target, set to IJBC or IJBB') +parser.add_argument('--target', + default='IJBC', + type=str, + help='target, set to IJBC or IJBB') args = parser.parse_args() target = args.target @@ -54,11 +57,11 @@ use_flip_test = True # if Ture, TestMode(F1) job = args.job batch_size = args.batch_size - # initialize Horovod # hvd.init() # rank_size = hvd.size() + # 将一个list尽量均分成n份，限制len(list)==n，份数大于原list内元素个数则分配空list[] def divideIntoNstrand(listTemp, n): twoList = [[] for i in range(n)] @@ -114,36 +117,40 @@ def get_image_feature(img_path, files_list, model_path, epoch, gpu_id): batch_data = np.empty((2 * batch_size, 3, 112, 112)) embedding = Embedding(model_path, epoch, data_shape, batch_size, gpu_id) - for img_index, each_line in enumerate(files[:len(files)-rare_size]): + for img_index, each_line in enumerate(files[:len(files) - rare_size]): name_lmk_score = each_line.strip().split(' ') img_name = os.path.join(img_path, name_lmk_score[0]) img = cv2.imread(img_name) - lmk = np.array([float(x) for x in name_lmk_score[1:-1]], dtype=np.float32) + lmk = np.array([float(x) for x in name_lmk_score[1:-1]], + dtype=np.float32) lmk = lmk.reshape((5, 2)) input_blob = embedding.get(img, lmk) # print(2*(img_index-batch*batch_size), 2*(img_index-batch*batch_size)+1) - batch_data[2*(img_index-batch*batch_size)][:] = input_blob[0] - batch_data[2*(img_index-batch*batch_size)+1][:] = input_blob[1] - if (img_index+1) % batch_size == 0: + batch_data[2 * (img_index - batch * batch_size)][:] = input_blob[0] + batch_data[2 * (img_index - batch * batch_size) + 1][:] = input_blob[1] + if (img_index + 1) % batch_size == 0: print('batch', batch) - img_feats[batch*batch_size:batch*batch_size+batch_size][:] = embedding.forward_db(batch_data) + img_feats[batch * batch_size:batch * batch_size + + batch_size][:] = embedding.forward_db(batch_data) batch += 1 faceness_scores.append(name_lmk_score[-1]) # img_feats = np.array(img_feats).astype(np.float32) batch_data = np.empty((2 * rare_size, 3, 112, 112)) embedding = Embedding(model_path, epoch, data_shape, rare_size, gpu_id) - for img_index, each_line in enumerate(files[len(files)-rare_size:]): + for img_index, each_line in enumerate(files[len(files) - rare_size:]): name_lmk_score = each_line.strip().split(' ') img_name = os.path.join(img_path, name_lmk_score[0]) img = cv2.imread(img_name) - lmk = np.array([float(x) for x in name_lmk_score[1:-1]], dtype=np.float32) + lmk = np.array([float(x) for x in name_lmk_score[1:-1]], + dtype=np.float32) lmk = lmk.reshape((5, 2)) input_blob = embedding.get(img, lmk) - batch_data[2*img_index][:] = input_blob[0] - batch_data[2*img_index+1][:] = input_blob[1] + batch_data[2 * img_index][:] = input_blob[0] + batch_data[2 * img_index + 1][:] = input_blob[1] if (img_index + 1) % rare_size == 0: print('batch', batch) - img_feats[len(files)-rare_size:][:] = embedding.forward_db(batch_data) + img_feats[len(files) - + rare_size:][:] = embedding.forward_db(batch_data) batch += 1 faceness_scores.append(name_lmk_score[-1]) faceness_scores = np.array(faceness_scores).astype(np.float32) @@ -151,6 +158,7 @@ def get_image_feature(img_path, files_list, model_path, epoch, gpu_id): # faceness_scores = np.ones( (len(files), ), dtype=np.float32 ) return img_feats, faceness_scores + # In[ ]: @@ -165,22 +173,26 @@ def image2template_feature(img_feats=None, templates=None, medias=None): for count_template, uqt in enumerate(unique_templates): - (ind_t,) = np.where(templates == uqt) + (ind_t, ) = np.where(templates == uqt) face_norm_feats = img_feats[ind_t] face_medias = medias[ind_t] - unique_medias, unique_media_counts = np.unique(face_medias, return_counts=True) + unique_medias, unique_media_counts = np.unique(face_medias, + return_counts=True) media_norm_feats = [] for u, ct in zip(unique_medias, unique_media_counts): - (ind_m,) = np.where(face_medias == u) + (ind_m, ) = np.where(face_medias == u) if ct == 1: media_norm_feats += [face_norm_feats[ind_m]] else: # image features from the same video will be aggregated into one feature - media_norm_feats += [np.mean(face_norm_feats[ind_m], axis=0, keepdims=True)] + media_norm_feats += [ + np.mean(face_norm_feats[ind_m], axis=0, keepdims=True) + ] media_norm_feats = np.array(media_norm_feats) # media_norm_feats = media_norm_feats / np.sqrt(np.sum(media_norm_feats ** 2, -1, keepdims=True)) template_feats[count_template] = np.sum(media_norm_feats, axis=0) if count_template % 2000 == 0: - print('Finish Calculating {} template features.'.format(count_template)) + print('Finish Calculating {} template features.'.format( + count_template)) # template_norm_feats = template_feats / np.sqrt(np.sum(template_feats ** 2, -1, keepdims=True)) template_norm_feats = sklearn.preprocessing.normalize(template_feats) # print(template_norm_feats.shape) @@ -190,7 +202,10 @@ def image2template_feature(img_feats=None, templates=None, medias=None): # In[ ]: -def verification(template_norm_feats=None, unique_templates=None, p1=None, p2=None): +def verification(template_norm_feats=None, + unique_templates=None, + p1=None, + p2=None): # ========================================================== # Compute set-to-set Similarity Score. # ========================================================== @@ -198,11 +213,13 @@ def verification(template_norm_feats=None, unique_templates=None, p1=None, p2=No for count_template, uqt in enumerate(unique_templates): template2id[uqt] = count_template - score = np.zeros((len(p1),)) # save cosine distance between pairs + score = np.zeros((len(p1), )) # save cosine distance between pairs total_pairs = np.array(range(len(p1))) batchsize = 100000 # small batchsize instead of all pairs in one batch due to the memory limiation - sublists = [total_pairs[i:i + batchsize] for i in range(0, len(p1), batchsize)] + sublists = [ + total_pairs[i:i + batchsize] for i in range(0, len(p1), batchsize) + ] total_sublists = len(sublists) for c, s in enumerate(sublists): feat1 = template_norm_feats[template2id[p1[s]]] @@ -215,14 +232,19 @@ def verification(template_norm_feats=None, unique_templates=None, p1=None, p2=No # In[ ]: -def verification2(template_norm_feats=None, unique_templates=None, p1=None, p2=None): +def verification2(template_norm_feats=None, + unique_templates=None, + p1=None, + p2=None): template2id = np.zeros((max(unique_templates) + 1, 1), dtype=int) for count_template, uqt in enumerate(unique_templates): template2id[uqt] = count_template - score = np.zeros((len(p1),)) # save cosine distance between pairs + score = np.zeros((len(p1), )) # save cosine distance between pairs total_pairs = np.array(range(len(p1))) batchsize = 100000 # small batchsize instead of all pairs in one batch due to the memory limiation - sublists = [total_pairs[i:i + batchsize] for i in range(0, len(p1), batchsize)] + sublists = [ + total_pairs[i:i + batchsize] for i in range(0, len(p1), batchsize) + ] total_sublists = len(sublists) for c, s in enumerate(sublists): feat1 = template_norm_feats[template2id[p1[s]]] @@ -254,13 +276,13 @@ assert target == 'IJBC' or target == 'IJBB' # ============================================================= start = timeit.default_timer() templates, medias = read_template_media_list( - os.path.join('%s/meta' % image_path, '%s_face_tid_mid.txt' % target.lower())) + os.path.join('%s/meta' % image_path, + '%s_face_tid_mid.txt' % target.lower())) stop = timeit.default_timer() print('Time: %.2f s. ' % (stop - start)) # In[ ]: - # ============================================================= # load template pairs for template-to-template verification # tid : template id, label : 1/0 @@ -269,7 +291,8 @@ print('Time: %.2f s. ' % (stop - start)) # ============================================================= start = timeit.default_timer() p1, p2, label = read_template_pair_list( - os.path.join('%s/meta' % image_path, '%s_template_pair_label.txt' % target.lower())) + os.path.join('%s/meta' % image_path, + '%s_template_pair_label.txt' % target.lower())) stop = timeit.default_timer() print('Time: %.2f s. ' % (stop - start)) @@ -277,7 +300,6 @@ print('Time: %.2f s. ' % (stop - start)) # In[ ]: - # ============================================================= # load image features # format: @@ -293,16 +315,17 @@ files_list = files # img_feats # for i in range(rank_size): -img_feats, faceness_scores = get_image_feature(img_path, files_list, model_path, epoch, gpu_id) +img_feats, faceness_scores = get_image_feature(img_path, files_list, + model_path, epoch, gpu_id) stop = timeit.default_timer() print('Time: %.2f s. ' % (stop - start)) -print('Feature Shape: ({} , {}) .'.format(img_feats.shape[0], img_feats.shape[1])) +print('Feature Shape: ({} , {}) .'.format(img_feats.shape[0], + img_feats.shape[1])) # # Step3: Get Template Features # In[ ]: - # ============================================================= # compute template features from image features. # ============================================================= @@ -314,12 +337,12 @@ start = timeit.default_timer() # 1. FaceScore （Feature Norm） # 2. FaceScore （Detector） - if use_flip_test: # concat --- F1 # img_input_feats = img_feats # add --- F2 - img_input_feats = img_feats[:, 0:img_feats.shape[1] // 2] + img_feats[:, img_feats.shape[1] // 2:] + img_input_feats = img_feats[:, 0:img_feats.shape[1] // + 2] + img_feats[:, img_feats.shape[1] // 2:] else: img_input_feats = img_feats[:, 0:img_feats.shape[1] // 2] @@ -327,7 +350,8 @@ if use_norm_score: img_input_feats = img_input_feats else: # normalise features to remove norm information - img_input_feats = img_input_feats / np.sqrt(np.sum(img_input_feats ** 2, -1, keepdims=True)) + img_input_feats = img_input_feats / np.sqrt( + np.sum(img_input_feats**2, -1, keepdims=True)) if use_detector_score: print(img_input_feats.shape, faceness_scores.shape) @@ -336,7 +360,8 @@ if use_detector_score: else: img_input_feats = img_input_feats -template_norm_feats, unique_templates = image2template_feature(img_input_feats, templates, medias) +template_norm_feats, unique_templates = image2template_feature( + img_input_feats, templates, medias) stop = timeit.default_timer() print('Time: %.2f s. ' % (stop - start)) @@ -344,7 +369,6 @@ print('Time: %.2f s. ' % (stop - start)) # In[ ]: - # ============================================================= # compute verification scores between template pairs. # ============================================================= @@ -367,7 +391,6 @@ np.save(score_save_file, score) # In[ ]: - files = [score_save_file] methods = [] scores = [] @@ -377,9 +400,10 @@ for file in files: methods = np.array(methods) scores = dict(zip(methods, scores)) -colours = dict(zip(methods, sample_colours_from_colourmap(methods.shape[0], 'Set2'))) +colours = dict( + zip(methods, sample_colours_from_colourmap(methods.shape[0], 'Set2'))) # x_labels = [1/(10**x) for x in np.linspace(6, 0, 6)] -x_labels = [10 ** -6, 10 ** -5, 10 ** -4, 10 ** -3, 10 ** -2, 10 ** -1] +x_labels = [10**-6, 10**-5, 10**-4, 10**-3, 10**-2, 10**-1] tpr_fpr_table = PrettyTable(['Methods'] + [str(x) for x in x_labels]) fig = plt.figure() for method in methods: @@ -387,16 +411,21 @@ for method in methods: roc_auc = auc(fpr, tpr) fpr = np.flipud(fpr) tpr = np.flipud(tpr) # select largest tpr at same fpr - plt.plot(fpr, tpr, color=colours[method], lw=1, - label=('[%s (AUC = %0.4f %%)]' % (method.split('-')[-1], roc_auc * 100))) + plt.plot(fpr, + tpr, + color=colours[method], + lw=1, + label=('[%s (AUC = %0.4f %%)]' % + (method.split('-')[-1], roc_auc * 100))) tpr_fpr_row = [] tpr_fpr_row.append("%s-%s" % (method, target)) for fpr_iter in np.arange(len(x_labels)): - _, min_index = min(list(zip(abs(fpr - x_labels[fpr_iter]), range(len(fpr))))) + _, min_index = min( + list(zip(abs(fpr - x_labels[fpr_iter]), range(len(fpr))))) # tpr_fpr_row.append('%.4f' % tpr[min_index]) tpr_fpr_row.append('%.2f' % (tpr[min_index] * 100)) tpr_fpr_table.add_row(tpr_fpr_row) -plt.xlim([10 ** -6, 0.1]) +plt.xlim([10**-6, 0.1]) plt.ylim([0.3, 1.0]) plt.grid(linestyle='--', linewidth=1) plt.xticks(x_labels) diff --git a/recognition/partial_fc/pytorch/IJB/IJB_11_Figure.py b/recognition/partial_fc/pytorch/IJB/IJB_11_Figure.py index 8a7270b..9e7852e 100644 --- a/recognition/partial_fc/pytorch/IJB/IJB_11_Figure.py +++ b/recognition/partial_fc/pytorch/IJB/IJB_11_Figure.py @@ -17,19 +17,29 @@ job = 'IJBC' title = 'IJB-C' root = '/train/trainset/1' -score_save_file_1 = '{}/glint-face/IJB/result/celeb360kfinal0.1/{}_result/cosface.npy'.format(root, target) -score_save_file_2 = '{}/glint-face/IJB/result/celeb360kfinal1.0/{}_result/cosface.npy'.format(root, target) -score_save_file_3 = '{}/glint-face/IJB/result/emore0.4/{}_result/cosface.npy'.format(root, target) -score_save_file_4 = '{}/glint-face/IJB/result/emore0.8/{}_result/cosface.npy'.format(root, target) -score_save_file_5 = '{}/glint-face/IJB/result/emore1.0/{}_result/cosface.npy'.format(root, target) -score_save_file_6 = '{}/glint-face/IJB/result/retina1.0/{}_result/cosface.npy'.format(root, target) - +score_save_file_1 = '{}/glint-face/IJB/result/celeb360kfinal0.1/{}_result/cosface.npy'.format( + root, target) +score_save_file_2 = '{}/glint-face/IJB/result/celeb360kfinal1.0/{}_result/cosface.npy'.format( + root, target) +score_save_file_3 = '{}/glint-face/IJB/result/emore0.4/{}_result/cosface.npy'.format( + root, target) +score_save_file_4 = '{}/glint-face/IJB/result/emore0.8/{}_result/cosface.npy'.format( + root, target) +score_save_file_5 = '{}/glint-face/IJB/result/emore1.0/{}_result/cosface.npy'.format( + root, target) +score_save_file_6 = '{}/glint-face/IJB/result/retina1.0/{}_result/cosface.npy'.format( + root, target) save_path = '{}/glint-face/IJB'.format(root) image_path = '{}/face/IJB_release/{}'.format(root, target) -methods = ['celeb360k_final0.1, S=0.1', 'celeb360k_final1.0, S=1.0', 'MS1MV2, S=0.4', 'MS1MV2, S=0.8', 'MS1MV2, S=1.0', 'RETINA, S=1.0'] -files = [score_save_file_1, score_save_file_2, score_save_file_3, score_save_file_4, score_save_file_5, - score_save_file_6] +methods = [ + 'celeb360k_final0.1, S=0.1', 'celeb360k_final1.0, S=1.0', 'MS1MV2, S=0.4', + 'MS1MV2, S=0.8', 'MS1MV2, S=1.0', 'RETINA, S=1.0' +] +files = [ + score_save_file_1, score_save_file_2, score_save_file_3, score_save_file_4, + score_save_file_5, score_save_file_6 +] def read_template_pair_list(path): @@ -43,8 +53,8 @@ def read_template_pair_list(path): p1, p2, label = read_template_pair_list( - os.path.join('%s/meta' % image_path, '%s_template_pair_label.txt' % target.lower())) - + os.path.join('%s/meta' % image_path, + '%s_template_pair_label.txt' % target.lower())) scores = [] for file in files: @@ -52,9 +62,10 @@ for file in files: methods = np.array(methods) scores = dict(zip(methods, scores)) -colours = dict(zip(methods, sample_colours_from_colourmap(methods.shape[0], 'Set2'))) +colours = dict( + zip(methods, sample_colours_from_colourmap(methods.shape[0], 'Set2'))) # x_labels = [1/(10**x) for x in np.linspace(6, 0, 6)] -x_labels = [10 ** -6, 10 ** -5, 10 ** -4, 10 ** -3, 10 ** -2, 10 ** -1] +x_labels = [10**-6, 10**-5, 10**-4, 10**-3, 10**-2, 10**-1] tpr_fpr_table = PrettyTable(['Methods'] + [str(x) for x in x_labels]) fig = plt.figure() for method in methods: @@ -62,17 +73,22 @@ for method in methods: roc_auc = auc(fpr, tpr) fpr = np.flipud(fpr) tpr = np.flipud(tpr) # select largest tpr at same fpr - plt.plot(fpr, tpr, color=colours[method], lw=1, - # label=('[%s (AUC = %0.4f %%)]' % (method.split('-')[-1], roc_auc * 100)) - label = method) + plt.plot( + fpr, + tpr, + color=colours[method], + lw=1, + # label=('[%s (AUC = %0.4f %%)]' % (method.split('-')[-1], roc_auc * 100)) + label=method) tpr_fpr_row = [] tpr_fpr_row.append("%s-%s" % (method, target)) for fpr_iter in np.arange(len(x_labels)): - _, min_index = min(list(zip(abs(fpr - x_labels[fpr_iter]), range(len(fpr))))) + _, min_index = min( + list(zip(abs(fpr - x_labels[fpr_iter]), range(len(fpr))))) # tpr_fpr_row.append('%.4f' % tpr[min_index]) tpr_fpr_row.append('%.2f' % (tpr[min_index] * 100)) tpr_fpr_table.add_row(tpr_fpr_row) -plt.xlim([10 ** -6, 0.1]) +plt.xlim([10**-6, 0.1]) plt.ylim([0.30, 1.0]) plt.grid(linestyle='--', linewidth=1) plt.xticks(x_labels) diff --git a/recognition/partial_fc/pytorch/IJB/IJB_11_test.py b/recognition/partial_fc/pytorch/IJB/IJB_11_test.py index 88da5ef..a4f9e9d 100644 --- a/recognition/partial_fc/pytorch/IJB/IJB_11_test.py +++ b/recognition/partial_fc/pytorch/IJB/IJB_11_test.py @@ -37,7 +37,10 @@ parser.add_argument('--image-path', default='', type=str, help='') parser.add_argument('--gpu', default=7, type=int, help='gpu id') parser.add_argument('--batch-size', default=32, type=int, help='') parser.add_argument('--job', default='insightface', type=str, help='job name') -parser.add_argument('--target', default='IJBC', type=str, help='target, set to IJBC or IJBB') +parser.add_argument('--target', + default='IJBC', + type=str, + help='target, set to IJBC or IJBB') args = parser.parse_args() target = args.target @@ -109,7 +112,8 @@ def get_image_feature(img_path, files_list, model_path, epoch, gpu_id): name_lmk_score = each_line.strip().split(' ') img_name = os.path.join(img_path, name_lmk_score[0]) img = cv2.imread(img_name) - lmk = np.array([float(x) for x in name_lmk_score[1:-1]], dtype=np.float32) + lmk = np.array([float(x) for x in name_lmk_score[1:-1]], + dtype=np.float32) lmk = lmk.reshape((5, 2)) img_feats.append(embedding.get(img, lmk)) faceness_scores.append(name_lmk_score[-1]) @@ -129,27 +133,31 @@ def image2template_feature(img_feats=None, templates=None, medias=None): # 1. face image feature l2 normalization. img_feats:[number_image x feats_dim] # 2. compute media feature. # 3. compute template feature. - # ========================================================== + # ========================================================== unique_templates = np.unique(templates) template_feats = np.zeros((len(unique_templates), img_feats.shape[1])) for count_template, uqt in enumerate(unique_templates): - (ind_t,) = np.where(templates == uqt) + (ind_t, ) = np.where(templates == uqt) face_norm_feats = img_feats[ind_t] face_medias = medias[ind_t] - unique_medias, unique_media_counts = np.unique(face_medias, return_counts=True) + unique_medias, unique_media_counts = np.unique(face_medias, + return_counts=True) media_norm_feats = [] for u, ct in zip(unique_medias, unique_media_counts): - (ind_m,) = np.where(face_medias == u) + (ind_m, ) = np.where(face_medias == u) if ct == 1: media_norm_feats += [face_norm_feats[ind_m]] else: # image features from the same video will be aggregated into one feature - media_norm_feats += [np.mean(face_norm_feats[ind_m], axis=0, keepdims=True)] + media_norm_feats += [ + np.mean(face_norm_feats[ind_m], axis=0, keepdims=True) + ] media_norm_feats = np.array(media_norm_feats) # media_norm_feats = media_norm_feats / np.sqrt(np.sum(media_norm_feats ** 2, -1, keepdims=True)) template_feats[count_template] = np.sum(media_norm_feats, axis=0) if count_template % 2000 == 0: - print('Finish Calculating {} template features.'.format(count_template)) + print('Finish Calculating {} template features.'.format( + count_template)) # template_norm_feats = template_feats / np.sqrt(np.sum(template_feats ** 2, -1, keepdims=True)) template_norm_feats = sklearn.preprocessing.normalize(template_feats) # print(template_norm_feats.shape) @@ -159,7 +167,10 @@ def image2template_feature(img_feats=None, templates=None, medias=None): # In[ ]: -def verification(template_norm_feats=None, unique_templates=None, p1=None, p2=None): +def verification(template_norm_feats=None, + unique_templates=None, + p1=None, + p2=None): # ========================================================== # Compute set-to-set Similarity Score. # ========================================================== @@ -167,11 +178,13 @@ def verification(template_norm_feats=None, unique_templates=None, p1=None, p2=No for count_template, uqt in enumerate(unique_templates): template2id[uqt] = count_template - score = np.zeros((len(p1),)) # save cosine distance between pairs + score = np.zeros((len(p1), )) # save cosine distance between pairs total_pairs = np.array(range(len(p1))) batchsize = 100000 # small batchsize instead of all pairs in one batch due to the memory limiation - sublists = [total_pairs[i:i + batchsize] for i in range(0, len(p1), batchsize)] + sublists = [ + total_pairs[i:i + batchsize] for i in range(0, len(p1), batchsize) + ] total_sublists = len(sublists) for c, s in enumerate(sublists): feat1 = template_norm_feats[template2id[p1[s]]] @@ -184,14 +197,19 @@ def verification(template_norm_feats=None, unique_templates=None, p1=None, p2=No # In[ ]: -def verification2(template_norm_feats=None, unique_templates=None, p1=None, p2=None): +def verification2(template_norm_feats=None, + unique_templates=None, + p1=None, + p2=None): template2id = np.zeros((max(unique_templates) + 1, 1), dtype=int) for count_template, uqt in enumerate(unique_templates): template2id[uqt] = count_template - score = np.zeros((len(p1),)) # save cosine distance between pairs + score = np.zeros((len(p1), )) # save cosine distance between pairs total_pairs = np.array(range(len(p1))) batchsize = 100000 # small batchsize instead of all pairs in one batch due to the memory limiation - sublists = [total_pairs[i:i + batchsize] for i in range(0, len(p1), batchsize)] + sublists = [ + total_pairs[i:i + batchsize] for i in range(0, len(p1), batchsize) + ] total_sublists = len(sublists) for c, s in enumerate(sublists): feat1 = template_norm_feats[template2id[p1[s]]] @@ -217,19 +235,19 @@ assert target == 'IJBC' or target == 'IJBB' # ============================================================= # load image and template relationships for template feature embedding -# tid --> template id, mid --> media id +# tid --> template id, mid --> media id # format: # image_name tid mid # ============================================================= start = timeit.default_timer() templates, medias = read_template_media_list( - os.path.join('%s/meta' % image_path, '%s_face_tid_mid.txt' % target.lower())) + os.path.join('%s/meta' % image_path, + '%s_face_tid_mid.txt' % target.lower())) stop = timeit.default_timer() print('Time: %.2f s. ' % (stop - start)) # In[ ]: - # ============================================================= # load template pairs for template-to-template verification # tid : template id, label : 1/0 @@ -238,7 +256,8 @@ print('Time: %.2f s. ' % (stop - start)) # ============================================================= start = timeit.default_timer() p1, p2, label = read_template_pair_list( - os.path.join('%s/meta' % image_path, '%s_template_pair_label.txt' % target.lower())) + os.path.join('%s/meta' % image_path, + '%s_template_pair_label.txt' % target.lower())) stop = timeit.default_timer() print('Time: %.2f s. ' % (stop - start)) @@ -246,9 +265,8 @@ print('Time: %.2f s. ' % (stop - start)) # In[ ]: - # ============================================================= -# load image features +# load image features # format: # img_feats: [image_num x feats_dim] (227630, 512) # ============================================================= @@ -261,33 +279,34 @@ files_list = divideIntoNstrand(files, rank_size) # img_feats # for i in range(rank_size): -img_feats, faceness_scores = get_image_feature(img_path, files_list, model_path, epoch, gpu_id) +img_feats, faceness_scores = get_image_feature(img_path, files_list, + model_path, epoch, gpu_id) stop = timeit.default_timer() print('Time: %.2f s. ' % (stop - start)) -print('Feature Shape: ({} , {}) .'.format(img_feats.shape[0], img_feats.shape[1])) +print('Feature Shape: ({} , {}) .'.format(img_feats.shape[0], + img_feats.shape[1])) # # Step3: Get Template Features # In[ ]: - # ============================================================= # compute template features from image features. # ============================================================= start = timeit.default_timer() -# ========================================================== +# ========================================================== # Norm feature before aggregation into template feature? # Feature norm from embedding network and faceness score are able to decrease weights for noise samples (not face). -# ========================================================== +# ========================================================== # 1. FaceScore （Feature Norm） # 2. FaceScore （Detector） - if use_flip_test: # concat --- F1 # img_input_feats = img_feats # add --- F2 - img_input_feats = img_feats[:, 0:img_feats.shape[1] // 2] + img_feats[:, img_feats.shape[1] // 2:] + img_input_feats = img_feats[:, 0:img_feats.shape[1] // + 2] + img_feats[:, img_feats.shape[1] // 2:] else: img_input_feats = img_feats[:, 0:img_feats.shape[1] // 2] @@ -295,7 +314,8 @@ if use_norm_score: img_input_feats = img_input_feats else: # normalise features to remove norm information - img_input_feats = img_input_feats / np.sqrt(np.sum(img_input_feats ** 2, -1, keepdims=True)) + img_input_feats = img_input_feats / np.sqrt( + np.sum(img_input_feats**2, -1, keepdims=True)) if use_detector_score: print(img_input_feats.shape, faceness_scores.shape) @@ -304,7 +324,8 @@ if use_detector_score: else: img_input_feats = img_input_feats -template_norm_feats, unique_templates = image2template_feature(img_input_feats, templates, medias) +template_norm_feats, unique_templates = image2template_feature( + img_input_feats, templates, medias) stop = timeit.default_timer() print('Time: %.2f s. ' % (stop - start)) @@ -312,7 +333,6 @@ print('Time: %.2f s. ' % (stop - start)) # In[ ]: - # ============================================================= # compute verification scores between template pairs. # ============================================================= @@ -335,7 +355,6 @@ np.save(score_save_file, score) # In[ ]: - files = [score_save_file] methods = [] scores = [] @@ -345,9 +364,10 @@ for file in files: methods = np.array(methods) scores = dict(zip(methods, scores)) -colours = dict(zip(methods, sample_colours_from_colourmap(methods.shape[0], 'Set2'))) +colours = dict( + zip(methods, sample_colours_from_colourmap(methods.shape[0], 'Set2'))) # x_labels = [1/(10**x) for x in np.linspace(6, 0, 6)] -x_labels = [10 ** -6, 10 ** -5, 10 ** -4, 10 ** -3, 10 ** -2, 10 ** -1] +x_labels = [10**-6, 10**-5, 10**-4, 10**-3, 10**-2, 10**-1] tpr_fpr_table = PrettyTable(['Methods'] + [str(x) for x in x_labels]) fig = plt.figure() for method in methods: @@ -355,16 +375,21 @@ for method in methods: roc_auc = auc(fpr, tpr) fpr = np.flipud(fpr) tpr = np.flipud(tpr) # select largest tpr at same fpr - plt.plot(fpr, tpr, color=colours[method], lw=1, - label=('[%s (AUC = %0.4f %%)]' % (method.split('-')[-1], roc_auc * 100))) + plt.plot(fpr, + tpr, + color=colours[method], + lw=1, + label=('[%s (AUC = %0.4f %%)]' % + (method.split('-')[-1], roc_auc * 100))) tpr_fpr_row = [] tpr_fpr_row.append("%s-%s" % (method, target)) for fpr_iter in np.arange(len(x_labels)): - _, min_index = min(list(zip(abs(fpr - x_labels[fpr_iter]), range(len(fpr))))) + _, min_index = min( + list(zip(abs(fpr - x_labels[fpr_iter]), range(len(fpr))))) # tpr_fpr_row.append('%.4f' % tpr[min_index]) tpr_fpr_row.append('%.2f' % (tpr[min_index] * 100)) tpr_fpr_table.add_row(tpr_fpr_row) -plt.xlim([10 ** -6, 0.1]) +plt.xlim([10**-6, 0.1]) plt.ylim([0.3, 1.0]) plt.grid(linestyle='--', linewidth=1) plt.xticks(x_labels) diff --git a/recognition/partial_fc/pytorch/IJB/IJB_1N.py b/recognition/partial_fc/pytorch/IJB/IJB_1N.py index e42b49b..85ea405 100644 --- a/recognition/partial_fc/pytorch/IJB/IJB_1N.py +++ b/recognition/partial_fc/pytorch/IJB/IJB_1N.py @@ -19,23 +19,26 @@ from embedding import Embedding from menpo.visualize import print_progress from menpo.visualize.viewmatplotlib import sample_colours_from_colourmap + def read_template_subject_id_list(path): ijb_meta = np.loadtxt(path, dtype=str, skiprows=1, delimiter=',') templates = ijb_meta[:, 0].astype(np.int) subject_ids = ijb_meta[:, 1].astype(np.int) return templates, subject_ids + def read_template_media_list(path): ijb_meta = np.loadtxt(path, dtype=str) - templates = ijb_meta[:,1].astype(np.int) - medias = ijb_meta[:,2].astype(np.int) + templates = ijb_meta[:, 1].astype(np.int) + medias = ijb_meta[:, 2].astype(np.int) return templates, medias + def read_template_pair_list(path): pairs = np.loadtxt(path, dtype=str) - t1 = pairs[:,0].astype(np.int) - t2 = pairs[:,1].astype(np.int) - label = pairs[:,2].astype(np.int) + t1 = pairs[:, 0].astype(np.int) + t2 = pairs[:, 1].astype(np.int) + label = pairs[:, 2].astype(np.int) return t1, t2, label @@ -51,14 +54,15 @@ def get_image_feature(img_path, img_list_path, model_path, epoch, gpu_id): faceness_scores = [] img_feats = [] for img_index, each_line in enumerate(files): - if img_index%500==0: - print('processing', img_index) + if img_index % 500 == 0: + print('processing', img_index) name_lmk_score = each_line.strip().split(' ') img_name = os.path.join(img_path, name_lmk_score[0]) img = cv2.imread(img_name) - lmk = np.array([float(x) for x in name_lmk_score[1:-1]], dtype=np.float32) - lmk = lmk.reshape( (5,2) ) - img_feats.append(embedding.get(img,lmk)) + lmk = np.array([float(x) for x in name_lmk_score[1:-1]], + dtype=np.float32) + lmk = lmk.reshape((5, 2)) + img_feats.append(embedding.get(img, lmk)) faceness_scores.append(name_lmk_score[-1]) img_feats = np.array(img_feats).astype(np.float32) faceness_scores = np.array(faceness_scores).astype(np.float32) @@ -67,49 +71,65 @@ def get_image_feature(img_path, img_list_path, model_path, epoch, gpu_id): #faceness_scores = np.ones( (len(files), ), dtype=np.float32 ) return img_feats, faceness_scores -def image2template_feature(img_feats = None, templates = None, medias = None, choose_templates = None, choose_ids = None): + +def image2template_feature(img_feats=None, + templates=None, + medias=None, + choose_templates=None, + choose_ids=None): # ========================================================== # 1. face image feature l2 normalization. img_feats:[number_image x feats_dim] # 2. compute media feature. # 3. compute template feature. - # ========================================================== + # ========================================================== unique_templates, indices = np.unique(choose_templates, return_index=True) unique_subjectids = choose_ids[indices] template_feats = np.zeros((len(unique_templates), img_feats.shape[1])) for count_template, uqt in enumerate(unique_templates): - (ind_t,) = np.where(templates == uqt) + (ind_t, ) = np.where(templates == uqt) face_norm_feats = img_feats[ind_t] face_medias = medias[ind_t] - unique_medias, unique_media_counts = np.unique(face_medias, return_counts=True) + unique_medias, unique_media_counts = np.unique(face_medias, + return_counts=True) media_norm_feats = [] - for u,ct in zip(unique_medias, unique_media_counts): - (ind_m,) = np.where(face_medias == u) + for u, ct in zip(unique_medias, unique_media_counts): + (ind_m, ) = np.where(face_medias == u) if ct == 1: media_norm_feats += [face_norm_feats[ind_m]] - else: # image features from the same video will be aggregated into one feature - media_norm_feats += [np.mean(face_norm_feats[ind_m], 0, keepdims=True)] + else: # image features from the same video will be aggregated into one feature + media_norm_feats += [ + np.mean(face_norm_feats[ind_m], 0, keepdims=True) + ] media_norm_feats = np.array(media_norm_feats) # media_norm_feats = media_norm_feats / np.sqrt(np.sum(media_norm_feats ** 2, -1, keepdims=True)) template_feats[count_template] = np.sum(media_norm_feats, 0) - if count_template % 2000 == 0: - print('Finish Calculating {} template features.'.format(count_template)) - template_norm_feats = template_feats / np.sqrt(np.sum(template_feats ** 2, -1, keepdims=True)) + if count_template % 2000 == 0: + print('Finish Calculating {} template features.'.format( + count_template)) + template_norm_feats = template_feats / np.sqrt( + np.sum(template_feats**2, -1, keepdims=True)) return template_norm_feats, unique_templates, unique_subjectids -def verification(template_norm_feats = None, unique_templates = None, p1 = None, p2 = None): + +def verification(template_norm_feats=None, + unique_templates=None, + p1=None, + p2=None): # ========================================================== # Compute set-to-set Similarity Score. # ========================================================== - template2id = np.zeros((max(unique_templates)+1,1),dtype=int) + template2id = np.zeros((max(unique_templates) + 1, 1), dtype=int) for count_template, uqt in enumerate(unique_templates): template2id[uqt] = count_template - - score = np.zeros((len(p1),)) # save cosine distance between pairs + + score = np.zeros((len(p1), )) # save cosine distance between pairs total_pairs = np.array(range(len(p1))) - batchsize = 100000 # small batchsize instead of all pairs in one batch due to the memory limiation - sublists = [total_pairs[i:i + batchsize] for i in range(0, len(p1), batchsize)] + batchsize = 100000 # small batchsize instead of all pairs in one batch due to the memory limiation + sublists = [ + total_pairs[i:i + batchsize] for i in range(0, len(p1), batchsize) + ] total_sublists = len(sublists) for c, s in enumerate(sublists): feat1 = template_norm_feats[template2id[p1[s]]] @@ -120,11 +140,13 @@ def verification(template_norm_feats = None, unique_templates = None, p1 = None, print('Finish {}/{} pairs.'.format(c, total_sublists)) return score + def read_score(path): with open(path, 'rb') as fid: img_feats = cPickle.load(fid) return img_feats + def evaluation(query_feats, gallery_feats, mask): Fars = [0.01, 0.1] print(query_feats.shape) @@ -138,28 +160,28 @@ def evaluation(query_feats, gallery_feats, mask): top_inds = np.argsort(-similarity) print(top_inds.shape) - # calculate top1 + # calculate top1 correct_num = 0 for i in range(query_num): j = top_inds[i, 0] if j == mask[i]: correct_num += 1 - print("top1 = {}".format(correct_num/query_num)) + print("top1 = {}".format(correct_num / query_num)) # calculate top5 correct_num = 0 for i in range(query_num): j = top_inds[i, 0:5] if mask[i] in j: correct_num += 1 - print("top5 = {}".format(correct_num/query_num)) + print("top5 = {}".format(correct_num / query_num)) # calculate 10 correct_num = 0 for i in range(query_num): j = top_inds[i, 0:10] if mask[i] in j: correct_num += 1 - print("top10 = {}".format(correct_num/query_num)) - + print("top10 = {}".format(correct_num / query_num)) + neg_pair_num = query_num * gallery_num - query_num print(neg_pair_num) required_topk = [math.ceil(query_num * x) for x in Fars] @@ -170,7 +192,7 @@ def evaluation(query_feats, gallery_feats, mask): gt = mask[i] pos_sims.append(top_sims[i, gt]) top_sims[i, gt] = -2.0 - + pos_sims = np.array(pos_sims) print(pos_sims.shape) neg_sims = top_sims[np.where(top_sims > -2.0)] @@ -178,137 +200,167 @@ def evaluation(query_feats, gallery_feats, mask): neg_sims = heapq.nlargest(max(required_topk), neg_sims) # heap sort print("after sorting , neg_sims num = {}".format(len(neg_sims))) for far, pos in zip(Fars, required_topk): - th = neg_sims[pos-1] + th = neg_sims[pos - 1] recall = np.sum(pos_sims > th) / query_num - print("far = {:.10f} pr = {:.10f} th = {:.10f}".format(far, recall, th)) - + print("far = {:.10f} pr = {:.10f} th = {:.10f}".format( + far, recall, th)) + + def gen_mask(query_ids, reg_ids): mask = [] for query_id in query_ids: pos = [i for i, x in enumerate(reg_ids) if query_id == x] if len(pos) != 1: - raise RuntimeError("RegIdsError with id = {}， duplicate = {} ".format(query_id, len(pos))) + raise RuntimeError( + "RegIdsError with id = {}， duplicate = {} ".format( + query_id, len(pos))) mask.append(pos[0]) return mask + if __name__ == "__main__": - parser = argparse.ArgumentParser(description='do ijb 1n test') - # general - parser.add_argument('--model-prefix', default='', help='path to load model.') - parser.add_argument('--model-epoch', default=1, type=int, help='') - parser.add_argument('--gpu', default=7, type=int, help='gpu id') - parser.add_argument('--batch-size', default=32, type=int, help='') - parser.add_argument('--job', default='insightface', type=str, help='job name') - parser.add_argument('--target', default='IJBC', type=str, help='target, set to IJBC or IJBB') - args = parser.parse_args() - target = args.target - model_path = args.model_prefix - gpu_id = args.gpu - epoch = args.model_epoch - meta_dir = "%s/meta"%args.target #meta root dir - if target=='IJBC': - gallery_s1_record = "%s_1N_gallery_G1.csv" % (args.target.lower()) - gallery_s2_record = "%s_1N_gallery_G2.csv" % (args.target.lower()) - else: - gallery_s1_record = "%s_1N_gallery_S1.csv" % (args.target.lower()) - gallery_s2_record = "%s_1N_gallery_S2.csv" % (args.target.lower()) - gallery_s1_templates, gallery_s1_subject_ids = read_template_subject_id_list(os.path.join(meta_dir, gallery_s1_record)) - print(gallery_s1_templates.shape, gallery_s1_subject_ids.shape) + parser = argparse.ArgumentParser(description='do ijb 1n test') + # general + parser.add_argument('--model-prefix', + default='', + help='path to load model.') + parser.add_argument('--model-epoch', default=1, type=int, help='') + parser.add_argument('--gpu', default=7, type=int, help='gpu id') + parser.add_argument('--batch-size', default=32, type=int, help='') + parser.add_argument('--job', + default='insightface', + type=str, + help='job name') + parser.add_argument('--target', + default='IJBC', + type=str, + help='target, set to IJBC or IJBB') + args = parser.parse_args() + target = args.target + model_path = args.model_prefix + gpu_id = args.gpu + epoch = args.model_epoch + meta_dir = "%s/meta" % args.target #meta root dir + if target == 'IJBC': + gallery_s1_record = "%s_1N_gallery_G1.csv" % (args.target.lower()) + gallery_s2_record = "%s_1N_gallery_G2.csv" % (args.target.lower()) + else: + gallery_s1_record = "%s_1N_gallery_S1.csv" % (args.target.lower()) + gallery_s2_record = "%s_1N_gallery_S2.csv" % (args.target.lower()) + gallery_s1_templates, gallery_s1_subject_ids = read_template_subject_id_list( + os.path.join(meta_dir, gallery_s1_record)) + print(gallery_s1_templates.shape, gallery_s1_subject_ids.shape) - gallery_s2_templates, gallery_s2_subject_ids = read_template_subject_id_list(os.path.join(meta_dir, gallery_s2_record)) - print(gallery_s2_templates.shape, gallery_s2_templates.shape) - - gallery_templates = np.concatenate([gallery_s1_templates, gallery_s2_templates]) - gallery_subject_ids = np.concatenate([gallery_s1_subject_ids, gallery_s2_subject_ids]) - print(gallery_templates.shape, gallery_subject_ids.shape) - - media_record = "%s_face_tid_mid.txt" % args.target.lower() - total_templates, total_medias = read_template_media_list(os.path.join(meta_dir, media_record)) - print("total_templates", total_templates.shape, total_medias.shape) - #load image features - start = timeit.default_timer() - feature_path = '' #feature path - face_path = '' #face path - img_path = './%s/loose_crop' % target - img_list_path = './%s/meta/%s_name_5pts_score.txt' % (target, target.lower()) - #img_feats, faceness_scores = get_image_feature(feature_path, face_path) - img_feats, faceness_scores = get_image_feature(img_path, img_list_path, model_path, epoch, gpu_id) - print('img_feats', img_feats.shape) - print('faceness_scores', faceness_scores.shape) - stop = timeit.default_timer() - print('Time: %.2f s. ' % (stop - start)) - print('Feature Shape: ({} , {}) .'.format(img_feats.shape[0], img_feats.shape[1])) - - # compute template features from image features. - start = timeit.default_timer() - # ========================================================== - # Norm feature before aggregation into template feature? - # Feature norm from embedding network and faceness score are able to decrease weights for noise samples (not face). - # ========================================================== - use_norm_score = True # if True, TestMode(N1) - use_detector_score = True # if True, TestMode(D1) - use_flip_test = True # if True, TestMode(F1) - - if use_flip_test: - # concat --- F1 - #img_input_feats = img_feats - # add --- F2 - img_input_feats = img_feats[:,0:int(img_feats.shape[1]/2)] + img_feats[:,int(img_feats.shape[1]/2):] - else: - img_input_feats = img_feats[:,0:int(img_feats.shape[1]/2)] - - if use_norm_score: - img_input_feats = img_input_feats - else: - # normalise features to remove norm information - img_input_feats = img_input_feats / np.sqrt(np.sum(img_input_feats ** 2, -1, keepdims=True)) - - if use_detector_score: - img_input_feats = img_input_feats * np.matlib.repmat(faceness_scores[:,np.newaxis], 1, img_input_feats.shape[1]) - else: - img_input_feats = img_input_feats - print("input features shape", img_input_feats.shape) - - #load gallery feature - gallery_templates_feature, gallery_unique_templates, gallery_unique_subject_ids = image2template_feature(img_input_feats, total_templates, total_medias, gallery_templates, gallery_subject_ids) - stop = timeit.default_timer() - print('Time: %.2f s. ' % (stop - start)) - print("gallery_templates_feature", gallery_templates_feature.shape) - print("gallery_unique_subject_ids", gallery_unique_subject_ids.shape) - #np.savetxt("gallery_templates_feature.txt", gallery_templates_feature) - #np.savetxt("gallery_unique_subject_ids.txt", gallery_unique_subject_ids) + gallery_s2_templates, gallery_s2_subject_ids = read_template_subject_id_list( + os.path.join(meta_dir, gallery_s2_record)) + print(gallery_s2_templates.shape, gallery_s2_templates.shape) - #load prope feature - probe_mixed_record = "%s_1N_probe_mixed.csv" % target.lower() - probe_mixed_templates, probe_mixed_subject_ids = read_template_subject_id_list(os.path.join(meta_dir, probe_mixed_record)) - print(probe_mixed_templates.shape, probe_mixed_subject_ids.shape) - probe_mixed_templates_feature, probe_mixed_unique_templates, probe_mixed_unique_subject_ids = image2template_feature(img_input_feats, total_templates, total_medias, probe_mixed_templates, probe_mixed_subject_ids) - print("probe_mixed_templates_feature", probe_mixed_templates_feature.shape) - print("probe_mixed_unique_subject_ids", probe_mixed_unique_subject_ids.shape) - #np.savetxt("probe_mixed_templates_feature.txt", probe_mixed_templates_feature) - #np.savetxt("probe_mixed_unique_subject_ids.txt", probe_mixed_unique_subject_ids) + gallery_templates = np.concatenate( + [gallery_s1_templates, gallery_s2_templates]) + gallery_subject_ids = np.concatenate( + [gallery_s1_subject_ids, gallery_s2_subject_ids]) + print(gallery_templates.shape, gallery_subject_ids.shape) - #root_dir = "" #feature root dir - #gallery_id_path = "" #id filepath - #gallery_feats_path = "" #feature filelpath - #print("{}: start loading gallery feat {}".format(dt.now(), gallery_id_path)) - #gallery_ids, gallery_feats = load_feat_file(root_dir, gallery_id_path, gallery_feats_path) - #print("{}: end loading gallery feat".format(dt.now())) - # - #probe_id_path = "probe_mixed_unique_subject_ids.txt" #probe id filepath - #probe_feats_path = "probe_mixed_templates_feature.txt" #probe feats filepath - #print("{}: start loading probe feat {}".format(dt.now(), probe_id_path)) - #probe_ids, probe_feats = load_feat_file(root_dir, probe_id_path, probe_feats_path) - #print("{}: end loading probe feat".format(dt.now())) + media_record = "%s_face_tid_mid.txt" % args.target.lower() + total_templates, total_medias = read_template_media_list( + os.path.join(meta_dir, media_record)) + print("total_templates", total_templates.shape, total_medias.shape) + #load image features + start = timeit.default_timer() + feature_path = '' #feature path + face_path = '' #face path + img_path = './%s/loose_crop' % target + img_list_path = './%s/meta/%s_name_5pts_score.txt' % (target, + target.lower()) + #img_feats, faceness_scores = get_image_feature(feature_path, face_path) + img_feats, faceness_scores = get_image_feature(img_path, img_list_path, + model_path, epoch, gpu_id) + print('img_feats', img_feats.shape) + print('faceness_scores', faceness_scores.shape) + stop = timeit.default_timer() + print('Time: %.2f s. ' % (stop - start)) + print('Feature Shape: ({} , {}) .'.format(img_feats.shape[0], + img_feats.shape[1])) - gallery_ids = gallery_unique_subject_ids - gallery_feats = gallery_templates_feature - probe_ids = probe_mixed_unique_subject_ids - probe_feats = probe_mixed_templates_feature + # compute template features from image features. + start = timeit.default_timer() + # ========================================================== + # Norm feature before aggregation into template feature? + # Feature norm from embedding network and faceness score are able to decrease weights for noise samples (not face). + # ========================================================== + use_norm_score = True # if True, TestMode(N1) + use_detector_score = True # if True, TestMode(D1) + use_flip_test = True # if True, TestMode(F1) - mask = gen_mask(probe_ids, gallery_ids) - - print("{}: start evaluation".format(dt.now())) - evaluation(probe_feats, gallery_feats, mask) - print("{}: end evaluation".format(dt.now())) + if use_flip_test: + # concat --- F1 + #img_input_feats = img_feats + # add --- F2 + img_input_feats = img_feats[:, 0:int( + img_feats.shape[1] / 2)] + img_feats[:, + int(img_feats.shape[1] / 2):] + else: + img_input_feats = img_feats[:, 0:int(img_feats.shape[1] / 2)] + if use_norm_score: + img_input_feats = img_input_feats + else: + # normalise features to remove norm information + img_input_feats = img_input_feats / np.sqrt( + np.sum(img_input_feats**2, -1, keepdims=True)) + + if use_detector_score: + img_input_feats = img_input_feats * np.matlib.repmat( + faceness_scores[:, np.newaxis], 1, img_input_feats.shape[1]) + else: + img_input_feats = img_input_feats + print("input features shape", img_input_feats.shape) + + #load gallery feature + gallery_templates_feature, gallery_unique_templates, gallery_unique_subject_ids = image2template_feature( + img_input_feats, total_templates, total_medias, gallery_templates, + gallery_subject_ids) + stop = timeit.default_timer() + print('Time: %.2f s. ' % (stop - start)) + print("gallery_templates_feature", gallery_templates_feature.shape) + print("gallery_unique_subject_ids", gallery_unique_subject_ids.shape) + #np.savetxt("gallery_templates_feature.txt", gallery_templates_feature) + #np.savetxt("gallery_unique_subject_ids.txt", gallery_unique_subject_ids) + + #load prope feature + probe_mixed_record = "%s_1N_probe_mixed.csv" % target.lower() + probe_mixed_templates, probe_mixed_subject_ids = read_template_subject_id_list( + os.path.join(meta_dir, probe_mixed_record)) + print(probe_mixed_templates.shape, probe_mixed_subject_ids.shape) + probe_mixed_templates_feature, probe_mixed_unique_templates, probe_mixed_unique_subject_ids = image2template_feature( + img_input_feats, total_templates, total_medias, probe_mixed_templates, + probe_mixed_subject_ids) + print("probe_mixed_templates_feature", probe_mixed_templates_feature.shape) + print("probe_mixed_unique_subject_ids", + probe_mixed_unique_subject_ids.shape) + #np.savetxt("probe_mixed_templates_feature.txt", probe_mixed_templates_feature) + #np.savetxt("probe_mixed_unique_subject_ids.txt", probe_mixed_unique_subject_ids) + + #root_dir = "" #feature root dir + #gallery_id_path = "" #id filepath + #gallery_feats_path = "" #feature filelpath + #print("{}: start loading gallery feat {}".format(dt.now(), gallery_id_path)) + #gallery_ids, gallery_feats = load_feat_file(root_dir, gallery_id_path, gallery_feats_path) + #print("{}: end loading gallery feat".format(dt.now())) + # + #probe_id_path = "probe_mixed_unique_subject_ids.txt" #probe id filepath + #probe_feats_path = "probe_mixed_templates_feature.txt" #probe feats filepath + #print("{}: start loading probe feat {}".format(dt.now(), probe_id_path)) + #probe_ids, probe_feats = load_feat_file(root_dir, probe_id_path, probe_feats_path) + #print("{}: end loading probe feat".format(dt.now())) + + gallery_ids = gallery_unique_subject_ids + gallery_feats = gallery_templates_feature + probe_ids = probe_mixed_unique_subject_ids + probe_feats = probe_mixed_templates_feature + + mask = gen_mask(probe_ids, gallery_ids) + + print("{}: start evaluation".format(dt.now())) + evaluation(probe_feats, gallery_feats, mask) + print("{}: end evaluation".format(dt.now())) diff --git a/recognition/partial_fc/pytorch/IJB/plot_ax.py b/recognition/partial_fc/pytorch/IJB/plot_ax.py index f6ea4da..2c11e81 100644 --- a/recognition/partial_fc/pytorch/IJB/plot_ax.py +++ b/recognition/partial_fc/pytorch/IJB/plot_ax.py @@ -21,34 +21,54 @@ root = '/train/trainset/1' #retina = '{}/glint-face/IJB/result/retina1.0/{}_result/cosface.npy'.format(root, target) #glint_retinaface_fp16 = '{}/glint-face/IJB/result/glint_retinaface_fp16/{}_result/cosface.npy'.format(root, target) -retina_fp16_10_percents = '{}/glint-face/IJB/result/glint_retinaface_fp16_0.1/{}_result/arcface.npy'.format(root, target) -retina_fp32_10_percents = '{}/glint-face/IJB/result/retina_0.1_fp32/{}_result/cosface.npy'.format(root, target) -retina_fp16 = '{}/glint-face/IJB/result/glint_retinaface_fp16/{}_result/cosface.npy'.format(root, target) -celeb360k_final = '{}/glint-face/IJB/result/celeb360kfinal1.0/{}_result/cosface.npy'.format(root, target) -celeb360k_final_10_percents = '{}/glint-face/IJB/result/celeb360kfinal0.1/{}_result/cosface.npy'.format(root, target) -retina_4GPU = '{}/glint-face/IJB/result/anxiang_ms1m_retina/{}_result/cosface.npy'.format(root, target) -retina_4GPU_scale2 = '{}/glint-face/IJB/result/anxiang_retina_largelr/{}_result/cosface.npy'.format(root, target) -emore_percents_10 = '{}/glint-face/IJB/result/emore0.1/{}_result/cosface.npy'.format(root, target) -emore_percents_40 = '{}/glint-face/IJB/result/emore0.4/{}_result/cosface.npy'.format(root, target) -emore_percents_80 = '{}/glint-face/IJB/result/emore0.8/{}_result/cosface.npy'.format(root, target) +retina_fp16_10_percents = '{}/glint-face/IJB/result/glint_retinaface_fp16_0.1/{}_result/arcface.npy'.format( + root, target) +retina_fp32_10_percents = '{}/glint-face/IJB/result/retina_0.1_fp32/{}_result/cosface.npy'.format( + root, target) +retina_fp16 = '{}/glint-face/IJB/result/glint_retinaface_fp16/{}_result/cosface.npy'.format( + root, target) +celeb360k_final = '{}/glint-face/IJB/result/celeb360kfinal1.0/{}_result/cosface.npy'.format( + root, target) +celeb360k_final_10_percents = '{}/glint-face/IJB/result/celeb360kfinal0.1/{}_result/cosface.npy'.format( + root, target) +retina_4GPU = '{}/glint-face/IJB/result/anxiang_ms1m_retina/{}_result/cosface.npy'.format( + root, target) +retina_4GPU_scale2 = '{}/glint-face/IJB/result/anxiang_retina_largelr/{}_result/cosface.npy'.format( + root, target) +emore_percents_10 = '{}/glint-face/IJB/result/emore0.1/{}_result/cosface.npy'.format( + root, target) +emore_percents_40 = '{}/glint-face/IJB/result/emore0.4/{}_result/cosface.npy'.format( + root, target) +emore_percents_80 = '{}/glint-face/IJB/result/emore0.8/{}_result/cosface.npy'.format( + root, target) #emore_percents_10 = '{}/glint-face/IJB/result/emore0.1/{}_result/cosface.npy'.format(root, target) #emore_percents_10 = '{}/glint-face/IJB/result/emore_cosface_0.1_margin0.45/{}_result/cosface.npy'.format(root, target) -emore = '{}/glint-face/IJB/result/emore1.0/{}_result/cosface.npy'.format(root, target) +emore = '{}/glint-face/IJB/result/emore1.0/{}_result/cosface.npy'.format( + root, target) #celeb360k_0_1 = '{}/glint-face/IJB/result/celeb360k_0.1/{}_result/cosface.npy'.format(root, target) #celeb360k_1_0_1 = '{}/glint-face/IJB/result/celeb360k/{}_result/cosface.npy'.format(root, target) save_path = '{}/glint-face/IJB'.format(root) image_path = '{}/face/IJB_release/{}'.format(root, target) -methods = ['retina_fp16', 'retina_fp16_0.1', 'retina_fp32_0.1', 'celeb360k_final', 'celeb360k_final_10_percents'] +methods = [ + 'retina_fp16', 'retina_fp16_0.1', 'retina_fp32_0.1', 'celeb360k_final', + 'celeb360k_final_10_percents' +] methods = ['retina_4GPU', 'retina_4GPU_scale2'] methods = ['emore', 'emore_percents_10'] -methods = ['emore', 'emore_percents_10', 'emore_percents_40', 'emore_percents_80'] -files = [retina_fp16, retina_fp16_10_percents, retina_fp32_10_percents, celeb360k_final, celeb360k_final_10_percents] +methods = [ + 'emore', 'emore_percents_10', 'emore_percents_40', 'emore_percents_80' +] +files = [ + retina_fp16, retina_fp16_10_percents, retina_fp32_10_percents, + celeb360k_final, celeb360k_final_10_percents +] #files = [retina_4GPU, retina_4GPU_scale2] #files = [emore, emore_percents_10] files = [emore, emore_percents_10, emore_percents_40, emore_percents_80] + def read_template_pair_list(path): pairs = pd.read_csv(path, sep=' ', header=None).values # print(pairs.shape) @@ -60,8 +80,8 @@ def read_template_pair_list(path): p1, p2, label = read_template_pair_list( - os.path.join('%s/meta' % image_path, '%s_template_pair_label.txt' % target.lower())) - + os.path.join('%s/meta' % image_path, + '%s_template_pair_label.txt' % target.lower())) scores = [] for file in files: @@ -69,9 +89,10 @@ for file in files: methods = np.array(methods) scores = dict(zip(methods, scores)) -colours = dict(zip(methods, sample_colours_from_colourmap(methods.shape[0], 'Set2'))) +colours = dict( + zip(methods, sample_colours_from_colourmap(methods.shape[0], 'Set2'))) # x_labels = [1/(10**x) for x in np.linspace(6, 0, 6)] -x_labels = [10 ** -6, 10 ** -5, 10 ** -4, 10 ** -3, 10 ** -2, 10 ** -1] +x_labels = [10**-6, 10**-5, 10**-4, 10**-3, 10**-2, 10**-1] tpr_fpr_table = PrettyTable(['Methods'] + [str(x) for x in x_labels]) fig = plt.figure() for method in methods: @@ -79,17 +100,22 @@ for method in methods: roc_auc = auc(fpr, tpr) fpr = np.flipud(fpr) tpr = np.flipud(tpr) # select largest tpr at same fpr - plt.plot(fpr, tpr, color=colours[method], lw=1, - # label=('[%s (AUC = %0.4f %%)]' % (method.split('-')[-1], roc_auc * 100)) - label = method) + plt.plot( + fpr, + tpr, + color=colours[method], + lw=1, + # label=('[%s (AUC = %0.4f %%)]' % (method.split('-')[-1], roc_auc * 100)) + label=method) tpr_fpr_row = [] tpr_fpr_row.append("%s-%s" % (method, target)) for fpr_iter in np.arange(len(x_labels)): - _, min_index = min(list(zip(abs(fpr - x_labels[fpr_iter]), range(len(fpr))))) + _, min_index = min( + list(zip(abs(fpr - x_labels[fpr_iter]), range(len(fpr))))) # tpr_fpr_row.append('%.4f' % tpr[min_index]) tpr_fpr_row.append('%.2f' % (tpr[min_index] * 100)) tpr_fpr_table.add_row(tpr_fpr_row) -plt.xlim([10 ** -6, 0.1]) +plt.xlim([10**-6, 0.1]) plt.ylim([0.30, 1.0]) plt.grid(linestyle='--', linewidth=1) plt.xticks(x_labels) diff --git a/recognition/partial_fc/pytorch/IJB/recognition/__init__.py b/recognition/partial_fc/pytorch/IJB/recognition/__init__.py index 9285caf..e69de29 100644 --- a/recognition/partial_fc/pytorch/IJB/recognition/__init__.py +++ b/recognition/partial_fc/pytorch/IJB/recognition/__init__.py @@ -1 +0,0 @@ -import * \ No newline at end of file diff --git a/recognition/partial_fc/pytorch/IJB/recognition/embedding_test.py b/recognition/partial_fc/pytorch/IJB/recognition/embedding_test.py index 17a6d97..bcf437d 100644 --- a/recognition/partial_fc/pytorch/IJB/recognition/embedding_test.py +++ b/recognition/partial_fc/pytorch/IJB/recognition/embedding_test.py @@ -7,14 +7,13 @@ import datetime from skimage import transform as trans import sklearn from sklearn import preprocessing -import torch +import torch from torchvision import transforms sys.path.append('/root/xy/work_dir/xyface/') -from backbones import iresnet50,iresnet100 +from backbones import iresnet50, iresnet100 from torch.nn.parallel import DistributedDataParallel - class Embedding: def __init__(self, prefix, epoch, data_shape, batch_size=1, ctx_id=0): print('loading', prefix, epoch) @@ -26,12 +25,10 @@ class Embedding: model = torch.nn.DataParallel(resnet) self.model = model self.model.eval() - src = np.array([ - [30.2946, 51.6963], - [65.5318, 51.5014], - [48.0252, 71.7366], - [33.5493, 92.3655], - [62.7299, 92.2041]], dtype=np.float32) + src = np.array( + [[30.2946, 51.6963], [65.5318, 51.5014], [48.0252, 71.7366], + [33.5493, 92.3655], [62.7299, 92.2041]], + dtype=np.float32) src[:, 0] += 8.0 self.src = src self.batch_size = batch_size @@ -53,18 +50,22 @@ class Embedding: tform = trans.SimilarityTransform() tform.estimate(landmark5, self.src) M = tform.params[0:2, :] - img = cv2.warpAffine(rimg, M, (self.image_size[1], self.image_size[0]), borderValue=0.0) + img = cv2.warpAffine(rimg, + M, (self.image_size[1], self.image_size[0]), + borderValue=0.0) img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) img_flip = np.fliplr(img) img = np.transpose(img, (2, 0, 1)) # 3*112*112, RGB img_flip = np.transpose(img_flip, (2, 0, 1)) - input_blob = np.zeros((2, 3, self.image_size[1], self.image_size[0]), dtype=np.uint8) + input_blob = np.zeros((2, 3, self.image_size[1], self.image_size[0]), + dtype=np.uint8) input_blob[0] = img input_blob[1] = img_flip return input_blob + @torch.no_grad() def forward_db(self, batch_data): - imgs=torch.Tensor(batch_data).cuda() + imgs = torch.Tensor(batch_data).cuda() imgs.div_(255).sub_(0.5).div_(0.5) feat = self.model(imgs) feat = feat.reshape([self.batch_size, 2 * feat.shape[1]]) @@ -75,7 +76,7 @@ if __name__ == "__main__": weight = torch.load('/root/xy/work_dir/xyface/backbone0.pth') resnet = resnet50().cuda() resnet.load_state_dict(weight) - res = torch.nn.DataParallel(resnet,[0,1,2,3,4,5,6,7]) - tin=torch.Tensor(1023,3,112,112).cuda() - out=res(tin) + res = torch.nn.DataParallel(resnet, [0, 1, 2, 3, 4, 5, 6, 7]) + tin = torch.Tensor(1023, 3, 112, 112).cuda() + out = res(tin) print(out.size()) diff --git a/recognition/partial_fc/pytorch/backbones/__init__.py b/recognition/partial_fc/pytorch/backbones/__init__.py index 671ed01..3196fef 100644 --- a/recognition/partial_fc/pytorch/backbones/__init__.py +++ b/recognition/partial_fc/pytorch/backbones/__init__.py @@ -1 +1 @@ -from .iresnet import iresnet34,iresnet50,iresnet100 \ No newline at end of file +from .iresnet import iresnet34, iresnet50, iresnet100 diff --git a/recognition/partial_fc/pytorch/backbones/iresnet.py b/recognition/partial_fc/pytorch/backbones/iresnet.py index 2c5b355..3c2eda2 100644 --- a/recognition/partial_fc/pytorch/backbones/iresnet.py +++ b/recognition/partial_fc/pytorch/backbones/iresnet.py @@ -13,32 +13,59 @@ model_urls = { def conv3x3(in_planes, out_planes, stride=1, groups=1, dilation=1): """3x3 convolution with padding""" - return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, - padding=dilation, groups=groups, bias=False, dilation=dilation) + return nn.Conv2d(in_planes, + out_planes, + kernel_size=3, + stride=stride, + padding=dilation, + groups=groups, + bias=False, + dilation=dilation) def conv1x1(in_planes, out_planes, stride=1): """1x1 convolution""" - return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False) + return nn.Conv2d(in_planes, + out_planes, + kernel_size=1, + stride=stride, + bias=False) class IBasicBlock(nn.Module): expansion = 1 - def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1, - base_width=64, dilation=1): + def __init__(self, + inplanes, + planes, + stride=1, + downsample=None, + groups=1, + base_width=64, + dilation=1): super(IBasicBlock, self).__init__() if groups != 1 or base_width != 64: - raise ValueError('BasicBlock only supports groups=1 and base_width=64') + raise ValueError( + 'BasicBlock only supports groups=1 and base_width=64') if dilation > 1: - raise NotImplementedError("Dilation > 1 not supported in BasicBlock") + raise NotImplementedError( + "Dilation > 1 not supported in BasicBlock") # Both self.conv1 and self.downsample layers downsample the input when stride != 1 - self.bn1 = nn.BatchNorm2d(inplanes, eps=1e-05, ) + self.bn1 = nn.BatchNorm2d( + inplanes, + eps=1e-05, + ) self.conv1 = conv3x3(inplanes, planes) - self.bn2 = nn.BatchNorm2d(planes, eps=1e-05, ) + self.bn2 = nn.BatchNorm2d( + planes, + eps=1e-05, + ) self.prelu = nn.PReLU(planes) self.conv2 = conv3x3(planes, planes, stride) - self.bn3 = nn.BatchNorm2d(planes, eps=1e-05, ) + self.bn3 = nn.BatchNorm2d( + planes, + eps=1e-05, + ) self.downsample = downsample self.stride = stride @@ -63,8 +90,14 @@ class IBasicBlock(nn.Module): class IResNet(nn.Module): fc_scale = 7 * 7 - def __init__(self, block, layers, num_features=512, zero_init_residual=False, - groups=1, width_per_group=64, replace_stride_with_dilation=None): + def __init__(self, + block, + layers, + num_features=512, + zero_init_residual=False, + groups=1, + width_per_group=64, + replace_stride_with_dilation=None): super(IResNet, self).__init__() self.inplanes = 64 @@ -75,30 +108,53 @@ class IResNet(nn.Module): replace_stride_with_dilation = [False, False, False] if len(replace_stride_with_dilation) != 3: raise ValueError("replace_stride_with_dilation should be None " - "or a 3-element tuple, got {}".format(replace_stride_with_dilation)) + "or a 3-element tuple, got {}".format( + replace_stride_with_dilation)) self.groups = groups self.base_width = width_per_group - self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=3, stride=1, padding=1, + self.conv1 = nn.Conv2d(3, + self.inplanes, + kernel_size=3, + stride=1, + padding=1, bias=False) self.bn1 = nn.BatchNorm2d(self.inplanes, eps=1e-05) self.prelu = nn.PReLU(self.inplanes) self.layer1 = self._make_layer(block, 64, layers[0], stride=2) - self.layer2 = self._make_layer(block, 128, layers[1], stride=2, + self.layer2 = self._make_layer(block, + 128, + layers[1], + stride=2, dilate=replace_stride_with_dilation[0]) - self.layer3 = self._make_layer(block, 256, layers[2], stride=2, + self.layer3 = self._make_layer(block, + 256, + layers[2], + stride=2, dilate=replace_stride_with_dilation[1]) - self.layer4 = self._make_layer(block, 512, layers[3], stride=2, + self.layer4 = self._make_layer(block, + 512, + layers[3], + stride=2, dilate=replace_stride_with_dilation[2]) self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) - self.bn2 = nn.BatchNorm2d(512 * block.expansion, eps=1e-05, ) + self.bn2 = nn.BatchNorm2d( + 512 * block.expansion, + eps=1e-05, + ) self.dropout = nn.Dropout(p=0.4, inplace=True) - self.fc = nn.Linear(512 * block.expansion * self.fc_scale, num_features) - self.features = nn.BatchNorm1d(num_features, eps=1e-05, ) + self.fc = nn.Linear(512 * block.expansion * self.fc_scale, + num_features) + self.features = nn.BatchNorm1d( + num_features, + eps=1e-05, + ) for m in self.modules(): if isinstance(m, nn.Conv2d): - nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') + nn.init.kaiming_normal_(m.weight, + mode='fan_out', + nonlinearity='relu') elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): nn.init.constant_(m.weight, 1) nn.init.constant_(m.bias, 0) @@ -117,16 +173,24 @@ class IResNet(nn.Module): if stride != 1 or self.inplanes != planes * block.expansion: downsample = nn.Sequential( conv1x1(self.inplanes, planes * block.expansion, stride), - nn.BatchNorm2d(planes * block.expansion, eps=1e-05, ), + nn.BatchNorm2d( + planes * block.expansion, + eps=1e-05, + ), ) layers = [] - layers.append(block(self.inplanes, planes, stride, downsample, self.groups, - self.base_width, previous_dilation)) + layers.append( + block(self.inplanes, planes, stride, downsample, self.groups, + self.base_width, previous_dilation)) self.inplanes = planes * block.expansion for _ in range(1, blocks): - layers.append(block(self.inplanes, planes, groups=self.groups, - base_width=self.base_width, dilation=self.dilation)) + layers.append( + block(self.inplanes, + planes, + groups=self.groups, + base_width=self.base_width, + dilation=self.dilation)) return nn.Sequential(*layers) @@ -159,15 +223,15 @@ def _iresnet(arch, block, layers, pretrained, progress, **kwargs): def iresnet34(pretrained=False, progress=True, **kwargs): - return _iresnet('iresnet34', IBasicBlock, [3, 4, 6, 3], pretrained, progress, - **kwargs) + return _iresnet('iresnet34', IBasicBlock, [3, 4, 6, 3], pretrained, + progress, **kwargs) def iresnet50(pretrained=False, progress=True, **kwargs): - return _iresnet('iresnet50', IBasicBlock, [3, 4, 14, 3], pretrained, progress, - **kwargs) + return _iresnet('iresnet50', IBasicBlock, [3, 4, 14, 3], pretrained, + progress, **kwargs) def iresnet100(pretrained=False, progress=True, **kwargs): - return _iresnet('iresnet100', IBasicBlock, [3, 13, 30, 3], pretrained, progress, - **kwargs) + return _iresnet('iresnet100', IBasicBlock, [3, 13, 30, 3], pretrained, + progress, **kwargs) diff --git a/recognition/partial_fc/pytorch/config.py b/recognition/partial_fc/pytorch/config.py index 0934181..1fc8b8f 100644 --- a/recognition/partial_fc/pytorch/config.py +++ b/recognition/partial_fc/pytorch/config.py @@ -15,10 +15,8 @@ if config.dataset == "emore": config.num_classes = 85742 config.num_epoch = 20 - def lr_step_func(epoch): - return ((epoch + 1) / (4 + 1)) ** 2 if epoch < -1 else 0.1 ** len( + return ((epoch + 1) / (4 + 1))**2 if epoch < -1 else 0.1**len( [m for m in [8, 16, 18] if m - 1 <= epoch]) - config.lr_func = lr_step_func diff --git a/recognition/partial_fc/pytorch/dataset.py b/recognition/partial_fc/pytorch/dataset.py index 49061d1..5608e2a 100644 --- a/recognition/partial_fc/pytorch/dataset.py +++ b/recognition/partial_fc/pytorch/dataset.py @@ -56,7 +56,8 @@ class DataLoaderX(DataLoader): return None with torch.cuda.stream(self.stream): for k in range(len(self.batch)): - self.batch[k] = self.batch[k].to(device=self.local_rank, non_blocking=True) + self.batch[k] = self.batch[k].to(device=self.local_rank, + non_blocking=True) def __next__(self): torch.cuda.current_stream().wait_stream(self.stream) @@ -83,8 +84,8 @@ class MXFaceDataset(Dataset): self.local_rank = local_rank path_imgrec = os.path.join(root_dir, 'train.rec') path_imgidx = os.path.join(root_dir, 'train.idx') - self.imgrec = mx.recordio.MXIndexedRecordIO( - path_imgidx, path_imgrec, 'r') + self.imgrec = mx.recordio.MXIndexedRecordIO(path_imgidx, path_imgrec, + 'r') s = self.imgrec.read_idx(0) header, _ = mx.recordio.unpack(s) if header.flag > 0: @@ -111,4 +112,3 @@ class MXFaceDataset(Dataset): def __len__(self): return len(self.imgidx) - diff --git a/recognition/partial_fc/pytorch/partial_classifier.py b/recognition/partial_fc/pytorch/partial_classifier.py index 64cbc12..33661af 100644 --- a/recognition/partial_fc/pytorch/partial_classifier.py +++ b/recognition/partial_fc/pytorch/partial_classifier.py @@ -26,13 +26,16 @@ class DistSampleClassifier(Module): def __init__(self, rank, local_rank, world_size): super(DistSampleClassifier, self).__init__() self.sample_rate = cfg.sample_rate - self.num_local = cfg.num_classes // world_size + int(rank < cfg.num_classes % world_size) - self.class_start = cfg.num_classes // world_size * rank + min(rank, cfg.num_classes % world_size) + self.num_local = cfg.num_classes // world_size + int( + rank < cfg.num_classes % world_size) + self.class_start = cfg.num_classes // world_size * rank + min( + rank, cfg.num_classes % world_size) self.num_sample = int(self.sample_rate * self.num_local) self.local_rank = local_rank self.world_size = world_size - self.weight = torch.empty(size=(self.num_local, cfg.embedding_size), device=local_rank) + self.weight = torch.empty(size=(self.num_local, cfg.embedding_size), + device=local_rank) self.weight_mom = torch.zeros_like(self.weight) self.stream = torch.cuda.Stream(local_rank) init.kaiming_uniform_(self.weight, a=math.sqrt(5)) @@ -48,7 +51,8 @@ class DistSampleClassifier(Module): @torch.no_grad() def sample(self, total_label): - P = (self.class_start <= total_label) & (total_label < self.class_start + self.num_local) + P = (self.class_start <= + total_label) & (total_label < self.class_start + self.num_local) total_label[~P] = -1 total_label[P] -= self.class_start if int(self.sample_rate) != 1: @@ -81,11 +85,15 @@ class DistSampleClassifier(Module): def prepare(self, label, optimizer): with torch.cuda.stream(self.stream): - total_label = torch.zeros(label.size()[0] * self.world_size, device=self.local_rank, dtype=torch.long) - dist.all_gather(list(total_label.chunk(self.world_size, dim=0)), label) + total_label = torch.zeros(label.size()[0] * self.world_size, + device=self.local_rank, + dtype=torch.long) + dist.all_gather(list(total_label.chunk(self.world_size, dim=0)), + label) self.sample(total_label) optimizer.state.pop(optimizer.param_groups[-1]['params'][0], None) optimizer.param_groups[-1]['params'][0] = self.sub_weight - optimizer.state[self.sub_weight]['momentum_buffer'] = self.sub_weight_mom + optimizer.state[ + self.sub_weight]['momentum_buffer'] = self.sub_weight_mom norm_weight = F.normalize(self.sub_weight) return total_label, norm_weight diff --git a/recognition/partial_fc/pytorch/partial_fc.py b/recognition/partial_fc/pytorch/partial_fc.py index fd361d7..a827d6f 100644 --- a/recognition/partial_fc/pytorch/partial_fc.py +++ b/recognition/partial_fc/pytorch/partial_fc.py @@ -32,7 +32,9 @@ class MarginSoftmax(nn.Module): def forward(self, cosine, label): index = torch.where(label != -1)[0] - m_hot = torch.zeros(index.size()[0], cosine.size()[1], device=cosine.device) + m_hot = torch.zeros(index.size()[0], + cosine.size()[1], + device=cosine.device) m_hot.scatter_(1, label[index, None], self.m) cosine[index] -= m_hot ret = cosine * self.s @@ -41,16 +43,24 @@ class MarginSoftmax(nn.Module): # ....... def main(local_rank, world_size, init_method='tcp://127.0.0.1:23499'): - dist.init_process_group(backend='nccl', init_method=init_method, rank=local_rank, world_size=world_size) + dist.init_process_group(backend='nccl', + init_method=init_method, + rank=local_rank, + world_size=world_size) cfg.local_rank = local_rank torch.cuda.set_device(local_rank) cfg.rank = dist.get_rank() cfg.world_size = world_size trainset = MXFaceDataset(root_dir=cfg.rec, local_rank=local_rank) - train_sampler = torch.utils.data.distributed.DistributedSampler(trainset, shuffle=True) + train_sampler = torch.utils.data.distributed.DistributedSampler( + trainset, shuffle=True) train_loader = DataLoaderX(local_rank=local_rank, - dataset=trainset, batch_size=cfg.batch_size, sampler=train_sampler, - num_workers=0, pin_memory=True, drop_last=False) + dataset=trainset, + batch_size=cfg.batch_size, + sampler=train_sampler, + num_workers=0, + pin_memory=True, + drop_last=False) backbone = backbones.iresnet100(False).to(local_rank) backbone.train() @@ -61,28 +71,31 @@ def main(local_rank, world_size, init_method='tcp://127.0.0.1:23499'): # DDP backbone = torch.nn.parallel.DistributedDataParallel( - module=backbone, - broadcast_buffers=False, - device_ids=[dist.get_rank()]) + module=backbone, broadcast_buffers=False, device_ids=[dist.get_rank()]) backbone.train() # Memory classifer - dist_sample_classifer = DistSampleClassifier( - rank=dist.get_rank(), - local_rank=local_rank, - world_size=world_size) + dist_sample_classifer = DistSampleClassifier(rank=dist.get_rank(), + local_rank=local_rank, + world_size=world_size) # Margin softmax margin_softmax = MarginSoftmax(s=64.0, m=0.4) # Optimizer for backbone and classifer - optimizer = SGD([ - {'params': backbone.parameters()}, - {'params': dist_sample_classifer.parameters()} - ], lr=0.1, momentum=0.9, weight_decay=cfg.weight_decay, rescale=world_size) + optimizer = SGD([{ + 'params': backbone.parameters() + }, { + 'params': dist_sample_classifer.parameters() + }], + lr=0.1, + momentum=0.9, + weight_decay=cfg.weight_decay, + rescale=world_size) # Lr scheduler - scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=cfg.lr_func) + scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, + lr_lambda=cfg.lr_func) n_epochs = cfg.num_epoch start_epoch = 0 @@ -94,12 +107,16 @@ def main(local_rank, world_size, init_method='tcp://127.0.0.1:23499'): train_sampler.set_epoch(epoch) for step, (img, label) in enumerate(train_loader): start = time.time() - total_label, norm_weight = dist_sample_classifer.prepare(label, optimizer) + total_label, norm_weight = dist_sample_classifer.prepare( + label, optimizer) features = F.normalize(backbone(img)) # Features all-gather - total_features = torch.zeros(features.size()[0] * world_size, cfg.embedding_size, device=local_rank) - dist.all_gather(list(total_features.chunk(world_size, dim=0)), features.data) + total_features = torch.zeros(features.size()[0] * world_size, + cfg.embedding_size, + device=local_rank) + dist.all_gather(list(total_features.chunk(world_size, dim=0)), + features.data) total_features.requires_grad = True # Calculate logits @@ -121,7 +138,9 @@ def main(local_rank, world_size, init_method='tcp://127.0.0.1:23499'): # Get one-hot grad = logits_exp index = torch.where(total_label != -1)[0] - one_hot = torch.zeros(index.size()[0], grad.size()[1], device=grad.device) + one_hot = torch.zeros(index.size()[0], + grad.size()[1], + device=grad.device) one_hot.scatter_(1, total_label[index, None], 1) # Calculate loss @@ -140,7 +159,8 @@ def main(local_rank, world_size, init_method='tcp://127.0.0.1:23499'): x_grad = torch.zeros_like(features) # Feature gradient all-reduce - dist.reduce_scatter(x_grad, list(total_features.grad.chunk(world_size, dim=0))) + dist.reduce_scatter( + x_grad, list(total_features.grad.chunk(world_size, dim=0))) # Backward backbone features.backward(x_grad) @@ -152,8 +172,10 @@ def main(local_rank, world_size, init_method='tcp://127.0.0.1:23499'): if cfg.rank == 0: writer.add_scalar('loss', loss_v, global_step) - print("Speed %d samples/sec Loss %.4f Epoch: %d Global Step: %d" % - ((cfg.batch_size / (time.time() - start) * world_size), loss_v, epoch, global_step)) + print( + "Speed %d samples/sec Loss %.4f Epoch: %d Global Step: %d" + % ((cfg.batch_size / (time.time() - start) * world_size), + loss_v, epoch, global_step)) global_step += 1 scheduler.step() @@ -161,7 +183,8 @@ def main(local_rank, world_size, init_method='tcp://127.0.0.1:23499'): import os if not os.path.exists('models'): os.makedirs('models') - torch.save(backbone.module.state_dict(), "models/" + str(epoch) + 'backbone.pth') + torch.save(backbone.module.state_dict(), + "models/" + str(epoch) + 'backbone.pth') dist.destroy_process_group() diff --git a/recognition/partial_fc/pytorch/sgd.py b/recognition/partial_fc/pytorch/sgd.py index 8a857fe..0ad91b0 100644 --- a/recognition/partial_fc/pytorch/sgd.py +++ b/recognition/partial_fc/pytorch/sgd.py @@ -3,21 +3,31 @@ from torch.optim.optimizer import Optimizer, required class SGD(Optimizer): - - def __init__(self, params, lr=required, momentum=0, dampening=0, - weight_decay=0, nesterov=False, rescale=1): + def __init__(self, + params, + lr=required, + momentum=0, + dampening=0, + weight_decay=0, + nesterov=False, + rescale=1): if lr is not required and lr < 0.0: raise ValueError("Invalid learning rate: {}".format(lr)) if momentum < 0.0: raise ValueError("Invalid momentum value: {}".format(momentum)) if weight_decay < 0.0: - raise ValueError("Invalid weight_decay value: {}".format(weight_decay)) + raise ValueError( + "Invalid weight_decay value: {}".format(weight_decay)) - defaults = dict(lr=lr, momentum=momentum, dampening=dampening, - weight_decay=weight_decay, nesterov=nesterov) + defaults = dict(lr=lr, + momentum=momentum, + dampening=dampening, + weight_decay=weight_decay, + nesterov=nesterov) self.rescale = rescale if nesterov and (momentum <= 0 or dampening != 0): - raise ValueError("Nesterov momentum requires a momentum and zero dampening") + raise ValueError( + "Nesterov momentum requires a momentum and zero dampening") super(SGD, self).__init__(params, defaults) def __setstate__(self, state): @@ -46,7 +56,8 @@ class SGD(Optimizer): if momentum != 0: param_state = self.state[p] if 'momentum_buffer' not in param_state: - buf = param_state['momentum_buffer'] = torch.clone(d_p).detach() + buf = param_state['momentum_buffer'] = torch.clone( + d_p).detach() else: buf = param_state['momentum_buffer'] buf.mul_(momentum).add_(other=d_p, alpha=1 - dampening) @@ -58,4 +69,3 @@ class SGD(Optimizer): p.data.add_(other=d_p, alpha=-group['lr']) return loss - diff --git a/recognition/partial_fc/unpack_glint360k.py b/recognition/partial_fc/unpack_glint360k.py index 7ac2e20..447e036 100644 --- a/recognition/partial_fc/unpack_glint360k.py +++ b/recognition/partial_fc/unpack_glint360k.py @@ -40,7 +40,8 @@ def main(args): if not os.path.exists(class_path): os.makedirs(class_path) _img = mx.image.imdecode(_img).asnumpy()[:, :, ::-1] # to bgr - image_path = os.path.join(class_path, "%d_%d.jpg" % (label, imgid)) + image_path = os.path.join(class_path, + "%d_%d.jpg" % (label, imgid)) cv2.imwrite(image_path, _img) imgid += 1 if imgid % 10000 == 0: diff --git a/recognition/symbol/fdensenet.py b/recognition/symbol/fdensenet.py index b37330e..b3d49ee 100644 --- a/recognition/symbol/fdensenet.py +++ b/recognition/symbol/fdensenet.py @@ -30,29 +30,35 @@ import symbol_utils sys.path.append(os.path.join(os.path.dirname(__file__), '..')) from config import config + def Act(): - if config.net_act=='prelu': - return nn.PReLU() + if config.net_act == 'prelu': + return nn.PReLU() else: - return nn.Activation(config.net_act) + return nn.Activation(config.net_act) + + # Helpers def _make_dense_block(num_layers, bn_size, growth_rate, dropout, stage_index): - out = nn.HybridSequential(prefix='stage%d_'%stage_index) + out = nn.HybridSequential(prefix='stage%d_' % stage_index) with out.name_scope(): for _ in range(num_layers): out.add(_make_dense_layer(growth_rate, bn_size, dropout)) return out + def _make_dense_layer(growth_rate, bn_size, dropout): new_features = nn.HybridSequential(prefix='') new_features.add(nn.BatchNorm()) #new_features.add(nn.Activation('relu')) new_features.add(Act()) - new_features.add(nn.Conv2D(bn_size * growth_rate, kernel_size=1, use_bias=False)) + new_features.add( + nn.Conv2D(bn_size * growth_rate, kernel_size=1, use_bias=False)) new_features.add(nn.BatchNorm()) #new_features.add(nn.Activation('relu')) new_features.add(Act()) - new_features.add(nn.Conv2D(growth_rate, kernel_size=3, padding=1, use_bias=False)) + new_features.add( + nn.Conv2D(growth_rate, kernel_size=3, padding=1, use_bias=False)) if dropout: new_features.add(nn.Dropout(dropout)) @@ -62,6 +68,7 @@ def _make_dense_layer(growth_rate, bn_size, dropout): return out + def _make_transition(num_output_features): out = nn.HybridSequential(prefix='') out.add(nn.BatchNorm()) @@ -71,6 +78,7 @@ def _make_transition(num_output_features): out.add(nn.AvgPool2D(pool_size=2, strides=2)) return out + # Net class DenseNet(nn.HybridBlock): r"""Densenet-BC model from the @@ -92,21 +100,33 @@ class DenseNet(nn.HybridBlock): classes : int, default 1000 Number of classification classes. """ - def __init__(self, num_init_features, growth_rate, block_config, - bn_size=4, dropout=0, classes=1000, **kwargs): + def __init__(self, + num_init_features, + growth_rate, + block_config, + bn_size=4, + dropout=0, + classes=1000, + **kwargs): super(DenseNet, self).__init__(**kwargs) with self.name_scope(): self.features = nn.HybridSequential(prefix='') - self.features.add(nn.Conv2D(num_init_features, kernel_size=3, - strides=1, padding=1, use_bias=False)) + self.features.add( + nn.Conv2D(num_init_features, + kernel_size=3, + strides=1, + padding=1, + use_bias=False)) self.features.add(nn.BatchNorm()) self.features.add(nn.Activation('relu')) self.features.add(nn.MaxPool2D(pool_size=3, strides=2, padding=1)) # Add dense blocks num_features = num_init_features for i, num_layers in enumerate(block_config): - self.features.add(_make_dense_block(num_layers, bn_size, growth_rate, dropout, i+1)) + self.features.add( + _make_dense_block(num_layers, bn_size, growth_rate, + dropout, i + 1)) num_features = num_features + num_layers * growth_rate if i != len(block_config) - 1: self.features.add(_make_transition(num_features // 2)) @@ -125,21 +145,25 @@ class DenseNet(nn.HybridBlock): # Specification -densenet_spec = {121: (64, 32, [6, 12, 24, 16]), - 161: (96, 48, [6, 12, 36, 24]), - 169: (64, 32, [6, 12, 32, 32]), - 201: (64, 32, [6, 12, 48, 32])} +densenet_spec = { + 121: (64, 32, [6, 12, 24, 16]), + 161: (96, 48, [6, 12, 36, 24]), + 169: (64, 32, [6, 12, 32, 32]), + 201: (64, 32, [6, 12, 48, 32]) +} # Constructor def get_symbol(): num_layers = config.num_layers num_init_features, growth_rate, block_config = densenet_spec[num_layers] - net = DenseNet(num_init_features, growth_rate, block_config, dropout = config.densenet_dropout) + net = DenseNet(num_init_features, + growth_rate, + block_config, + dropout=config.densenet_dropout) data = mx.sym.Variable(name='data') - data = data-127.5 - data = data*0.0078125 + data = data - 127.5 + data = data * 0.0078125 body = net(data) fc1 = symbol_utils.get_fc1(body, config.emb_size, config.net_output) return fc1 - diff --git a/recognition/symbol/fmnasnet.py b/recognition/symbol/fmnasnet.py index 9b0ba7f..118beb9 100644 --- a/recognition/symbol/fmnasnet.py +++ b/recognition/symbol/fmnasnet.py @@ -9,75 +9,103 @@ import symbol_utils sys.path.append(os.path.join(os.path.dirname(__file__), '..')) from config import config + def Act(): - if config.net_act=='prelu': - return nn.PReLU() + if config.net_act == 'prelu': + return nn.PReLU() else: - return nn.Activation(config.net_act) + return nn.Activation(config.net_act) + def ConvBlock(channels, kernel_size, strides, **kwargs): out = nn.HybridSequential(**kwargs) with out.name_scope(): out.add( - nn.Conv2D(channels, kernel_size, strides=strides, padding=1, use_bias=False), - nn.BatchNorm(scale=True), + nn.Conv2D(channels, + kernel_size, + strides=strides, + padding=1, + use_bias=False), nn.BatchNorm(scale=True), Act() #nn.Activation('relu') ) return out + def Conv1x1(channels, is_linear=False, **kwargs): out = nn.HybridSequential(**kwargs) with out.name_scope(): - out.add( - nn.Conv2D(channels, 1, padding=0, use_bias=False), - nn.BatchNorm(scale=True) - ) + out.add(nn.Conv2D(channels, 1, padding=0, use_bias=False), + nn.BatchNorm(scale=True)) if not is_linear: #out.add(nn.Activation('relu')) out.add(Act()) return out + def DWise(channels, strides, kernel_size=3, **kwargs): out = nn.HybridSequential(**kwargs) with out.name_scope(): out.add( - nn.Conv2D(channels, kernel_size, strides=strides, padding=kernel_size // 2, groups=channels, use_bias=False), - nn.BatchNorm(scale=True), + nn.Conv2D(channels, + kernel_size, + strides=strides, + padding=kernel_size // 2, + groups=channels, + use_bias=False), nn.BatchNorm(scale=True), Act() #nn.Activation('relu') ) return out + class SepCONV(nn.HybridBlock): - def __init__(self, inp, output, kernel_size, depth_multiplier=1, with_bn=True, **kwargs): + def __init__(self, + inp, + output, + kernel_size, + depth_multiplier=1, + with_bn=True, + **kwargs): super(SepCONV, self).__init__(**kwargs) with self.name_scope(): self.net = nn.HybridSequential() - cn = int(inp*depth_multiplier) + cn = int(inp * depth_multiplier) if output is None: self.net.add( - nn.Conv2D(in_channels=inp, channels=cn, groups=inp, kernel_size=kernel_size, strides=(1,1), padding=kernel_size // 2 - , use_bias=not with_bn) - ) + nn.Conv2D(in_channels=inp, + channels=cn, + groups=inp, + kernel_size=kernel_size, + strides=(1, 1), + padding=kernel_size // 2, + use_bias=not with_bn)) else: self.net.add( - nn.Conv2D(in_channels=inp, channels=cn, groups=inp, kernel_size=kernel_size, strides=(1,1), padding=kernel_size // 2 - , use_bias=False), + nn.Conv2D(in_channels=inp, + channels=cn, + groups=inp, + kernel_size=kernel_size, + strides=(1, 1), + padding=kernel_size // 2, + use_bias=False), nn.BatchNorm(), Act(), #nn.Activation('relu'), - nn.Conv2D(in_channels=cn, channels=output, kernel_size=(1,1), strides=(1,1) - , use_bias=not with_bn) - ) + nn.Conv2D(in_channels=cn, + channels=output, + kernel_size=(1, 1), + strides=(1, 1), + use_bias=not with_bn)) self.with_bn = with_bn self.act = Act() #self.act = nn.Activation('relu') if with_bn: self.bn = nn.BatchNorm() - def hybrid_forward(self, F ,x): + + def hybrid_forward(self, F, x): x = self.net(x) if self.with_bn: x = self.bn(x) @@ -85,25 +113,34 @@ class SepCONV(nn.HybridBlock): x = self.act(x) return x + class ExpandedConv(nn.HybridBlock): - def __init__(self, inp, oup, t, strides, kernel=3, same_shape=True, **kwargs): + def __init__(self, + inp, + oup, + t, + strides, + kernel=3, + same_shape=True, + **kwargs): super(ExpandedConv, self).__init__(**kwargs) self.same_shape = same_shape self.strides = strides - with self.name_scope(): + with self.name_scope(): self.bottleneck = nn.HybridSequential() self.bottleneck.add( - Conv1x1(inp*t, prefix="expand_"), - DWise(inp*t, self.strides, kernel, prefix="dwise_"), - Conv1x1(oup, is_linear=True, prefix="linear_") - ) + Conv1x1(inp * t, prefix="expand_"), + DWise(inp * t, self.strides, kernel, prefix="dwise_"), + Conv1x1(oup, is_linear=True, prefix="linear_")) + def hybrid_forward(self, F, x): out = self.bottleneck(x) if self.strides == 1 and self.same_shape: out = F.elemwise_add(out, x) return out + def ExpandedConvSequence(t, k, inp, oup, repeats, first_strides, **kwargs): seq = nn.HybridSequential(**kwargs) with seq.name_scope(): @@ -114,32 +151,40 @@ def ExpandedConvSequence(t, k, inp, oup, repeats, first_strides, **kwargs): curr_inp = oup return seq + class MNasNet(nn.HybridBlock): def __init__(self, m=1.0, **kwargs): super(MNasNet, self).__init__(**kwargs) - - self.first_oup = int(32*m) - self.second_oup = int(16*m) + + self.first_oup = int(32 * m) + self.second_oup = int(16 * m) #self.second_oup = int(32*m) self.interverted_residual_setting = [ # t, c, n, s, k - [3, int(24*m), 3, 2, 3, "stage2_"], # -> 56x56 - [3, int(40*m), 3, 2, 5, "stage3_"], # -> 28x28 - [6, int(80*m), 3, 2, 5, "stage4_1_"], # -> 14x14 - [6, int(96*m), 2, 1, 3, "stage4_2_"], # -> 14x14 - [6, int(192*m), 4, 2, 5, "stage5_1_"], # -> 7x7 - [6, int(320*m), 1, 1, 3, "stage5_2_"], # -> 7x7 + [3, int(24 * m), 3, 2, 3, "stage2_"], # -> 56x56 + [3, int(40 * m), 3, 2, 5, "stage3_"], # -> 28x28 + [6, int(80 * m), 3, 2, 5, "stage4_1_"], # -> 14x14 + [6, int(96 * m), 2, 1, 3, "stage4_2_"], # -> 14x14 + [6, int(192 * m), 4, 2, 5, "stage5_1_"], # -> 7x7 + [6, int(320 * m), 1, 1, 3, "stage5_2_"], # -> 7x7 ] - self.last_channels = int(1024*m) + self.last_channels = int(1024 * m) with self.name_scope(): self.features = nn.HybridSequential() - self.features.add(ConvBlock(self.first_oup, 3, 1, prefix="stage1_conv0_")) - self.features.add(SepCONV(self.first_oup, self.second_oup, 3, prefix="stage1_sepconv0_")) + self.features.add( + ConvBlock(self.first_oup, 3, 1, prefix="stage1_conv0_")) + self.features.add( + SepCONV(self.first_oup, + self.second_oup, + 3, + prefix="stage1_sepconv0_")) inp = self.second_oup - for i, (t, c, n, s, k, prefix) in enumerate(self.interverted_residual_setting): + for i, (t, c, n, s, k, + prefix) in enumerate(self.interverted_residual_setting): oup = c - self.features.add(ExpandedConvSequence(t, k, inp, oup, n, s, prefix=prefix)) + self.features.add( + ExpandedConvSequence(t, k, inp, oup, n, s, prefix=prefix)) inp = oup self.features.add(Conv1x1(self.last_channels, prefix="stage5_3_")) @@ -152,14 +197,17 @@ class MNasNet(nn.HybridBlock): return x def num_output_channel(self): - return self.last_channels + return self.last_channels + def get_symbol(): - net = MNasNet(config.net_multiplier) - data = mx.sym.Variable(name='data') - data = data-127.5 - data = data*0.0078125 - body = net(data) - fc1 = symbol_utils.get_fc1(body, config.emb_size, config.net_output, input_channel=net.num_output_channel()) - return fc1 - + net = MNasNet(config.net_multiplier) + data = mx.sym.Variable(name='data') + data = data - 127.5 + data = data * 0.0078125 + body = net(data) + fc1 = symbol_utils.get_fc1(body, + config.emb_size, + config.net_output, + input_channel=net.num_output_channel()) + return fc1 diff --git a/recognition/symbol/fmobilefacenet.py b/recognition/symbol/fmobilefacenet.py index 3050fc9..f498264 100644 --- a/recognition/symbol/fmobilefacenet.py +++ b/recognition/symbol/fmobilefacenet.py @@ -1,4 +1,3 @@ - import sys import os import mxnet as mx @@ -8,65 +7,218 @@ from config import config def Act(data, act_type, name): - #ignore param act_type, set it in this function - if act_type=='prelu': - body = mx.sym.LeakyReLU(data = data, act_type='prelu', name = name) + #ignore param act_type, set it in this function + if act_type == 'prelu': + body = mx.sym.LeakyReLU(data=data, act_type='prelu', name=name) else: - body = mx.sym.Activation(data=data, act_type=act_type, name=name) + body = mx.sym.Activation(data=data, act_type=act_type, name=name) return body -def Conv(data, num_filter=1, kernel=(1, 1), stride=(1, 1), pad=(0, 0), num_group=1, name=None, suffix=''): - conv = mx.sym.Convolution(data=data, num_filter=num_filter, kernel=kernel, num_group=num_group, stride=stride, pad=pad, no_bias=True, name='%s%s_conv2d' %(name, suffix)) - bn = mx.sym.BatchNorm(data=conv, name='%s%s_batchnorm' %(name, suffix), fix_gamma=False,momentum=config.bn_mom) - act = Act(data=bn, act_type=config.net_act, name='%s%s_relu' %(name, suffix)) + +def Conv(data, + num_filter=1, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + num_group=1, + name=None, + suffix=''): + conv = mx.sym.Convolution(data=data, + num_filter=num_filter, + kernel=kernel, + num_group=num_group, + stride=stride, + pad=pad, + no_bias=True, + name='%s%s_conv2d' % (name, suffix)) + bn = mx.sym.BatchNorm(data=conv, + name='%s%s_batchnorm' % (name, suffix), + fix_gamma=False, + momentum=config.bn_mom) + act = Act(data=bn, + act_type=config.net_act, + name='%s%s_relu' % (name, suffix)) return act - -def Linear(data, num_filter=1, kernel=(1, 1), stride=(1, 1), pad=(0, 0), num_group=1, name=None, suffix=''): - conv = mx.sym.Convolution(data=data, num_filter=num_filter, kernel=kernel, num_group=num_group, stride=stride, pad=pad, no_bias=True, name='%s%s_conv2d' %(name, suffix)) - bn = mx.sym.BatchNorm(data=conv, name='%s%s_batchnorm' %(name, suffix), fix_gamma=False,momentum=config.bn_mom) + + +def Linear(data, + num_filter=1, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + num_group=1, + name=None, + suffix=''): + conv = mx.sym.Convolution(data=data, + num_filter=num_filter, + kernel=kernel, + num_group=num_group, + stride=stride, + pad=pad, + no_bias=True, + name='%s%s_conv2d' % (name, suffix)) + bn = mx.sym.BatchNorm(data=conv, + name='%s%s_batchnorm' % (name, suffix), + fix_gamma=False, + momentum=config.bn_mom) return bn -def ConvOnly(data, num_filter=1, kernel=(1, 1), stride=(1, 1), pad=(0, 0), num_group=1, name=None, suffix=''): - conv = mx.sym.Convolution(data=data, num_filter=num_filter, kernel=kernel, num_group=num_group, stride=stride, pad=pad, no_bias=True, name='%s%s_conv2d' %(name, suffix)) - return conv - -def DResidual(data, num_out=1, kernel=(3, 3), stride=(2, 2), pad=(1, 1), num_group=1, name=None, suffix=''): - conv = Conv(data=data, num_filter=num_group, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name='%s%s_conv_sep' %(name, suffix)) - conv_dw = Conv(data=conv, num_filter=num_group, num_group=num_group, kernel=kernel, pad=pad, stride=stride, name='%s%s_conv_dw' %(name, suffix)) - proj = Linear(data=conv_dw, num_filter=num_out, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name='%s%s_conv_proj' %(name, suffix)) +def ConvOnly(data, + num_filter=1, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + num_group=1, + name=None, + suffix=''): + conv = mx.sym.Convolution(data=data, + num_filter=num_filter, + kernel=kernel, + num_group=num_group, + stride=stride, + pad=pad, + no_bias=True, + name='%s%s_conv2d' % (name, suffix)) + return conv + + +def DResidual(data, + num_out=1, + kernel=(3, 3), + stride=(2, 2), + pad=(1, 1), + num_group=1, + name=None, + suffix=''): + conv = Conv(data=data, + num_filter=num_group, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + name='%s%s_conv_sep' % (name, suffix)) + conv_dw = Conv(data=conv, + num_filter=num_group, + num_group=num_group, + kernel=kernel, + pad=pad, + stride=stride, + name='%s%s_conv_dw' % (name, suffix)) + proj = Linear(data=conv_dw, + num_filter=num_out, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + name='%s%s_conv_proj' % (name, suffix)) return proj - -def Residual(data, num_block=1, num_out=1, kernel=(3, 3), stride=(1, 1), pad=(1, 1), num_group=1, name=None, suffix=''): - identity=data + + +def Residual(data, + num_block=1, + num_out=1, + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + num_group=1, + name=None, + suffix=''): + identity = data for i in range(num_block): - shortcut=identity - conv=DResidual(data=identity, num_out=num_out, kernel=kernel, stride=stride, pad=pad, num_group=num_group, name='%s%s_block' %(name, suffix), suffix='%d'%i) - identity=conv+shortcut + shortcut = identity + conv = DResidual(data=identity, + num_out=num_out, + kernel=kernel, + stride=stride, + pad=pad, + num_group=num_group, + name='%s%s_block' % (name, suffix), + suffix='%d' % i) + identity = conv + shortcut return identity - + def get_symbol(): num_classes = config.emb_size print('in_network', config) fc_type = config.net_output data = mx.symbol.Variable(name="data") - data = data-127.5 - data = data*0.0078125 + data = data - 127.5 + data = data * 0.0078125 blocks = config.net_blocks - conv_1 = Conv(data, num_filter=64, kernel=(3, 3), pad=(1, 1), stride=(2, 2), name="conv_1") - if blocks[0]==1: - conv_2_dw = Conv(conv_1, num_group=64, num_filter=64, kernel=(3, 3), pad=(1, 1), stride=(1, 1), name="conv_2_dw") + conv_1 = Conv(data, + num_filter=64, + kernel=(3, 3), + pad=(1, 1), + stride=(2, 2), + name="conv_1") + if blocks[0] == 1: + conv_2_dw = Conv(conv_1, + num_group=64, + num_filter=64, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + name="conv_2_dw") else: - conv_2_dw = Residual(conv_1, num_block=blocks[0], num_out=64, kernel=(3, 3), stride=(1, 1), pad=(1, 1), num_group=64, name="res_2") - conv_23 = DResidual(conv_2_dw, num_out=64, kernel=(3, 3), stride=(2, 2), pad=(1, 1), num_group=128, name="dconv_23") - conv_3 = Residual(conv_23, num_block=blocks[1], num_out=64, kernel=(3, 3), stride=(1, 1), pad=(1, 1), num_group=128, name="res_3") - conv_34 = DResidual(conv_3, num_out=128, kernel=(3, 3), stride=(2, 2), pad=(1, 1), num_group=256, name="dconv_34") - conv_4 = Residual(conv_34, num_block=blocks[2], num_out=128, kernel=(3, 3), stride=(1, 1), pad=(1, 1), num_group=256, name="res_4") - conv_45 = DResidual(conv_4, num_out=128, kernel=(3, 3), stride=(2, 2), pad=(1, 1), num_group=512, name="dconv_45") - conv_5 = Residual(conv_45, num_block=blocks[3], num_out=128, kernel=(3, 3), stride=(1, 1), pad=(1, 1), num_group=256, name="res_5") - conv_6_sep = Conv(conv_5, num_filter=512, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_6sep") + conv_2_dw = Residual(conv_1, + num_block=blocks[0], + num_out=64, + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + num_group=64, + name="res_2") + conv_23 = DResidual(conv_2_dw, + num_out=64, + kernel=(3, 3), + stride=(2, 2), + pad=(1, 1), + num_group=128, + name="dconv_23") + conv_3 = Residual(conv_23, + num_block=blocks[1], + num_out=64, + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + num_group=128, + name="res_3") + conv_34 = DResidual(conv_3, + num_out=128, + kernel=(3, 3), + stride=(2, 2), + pad=(1, 1), + num_group=256, + name="dconv_34") + conv_4 = Residual(conv_34, + num_block=blocks[2], + num_out=128, + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + num_group=256, + name="res_4") + conv_45 = DResidual(conv_4, + num_out=128, + kernel=(3, 3), + stride=(2, 2), + pad=(1, 1), + num_group=512, + name="dconv_45") + conv_5 = Residual(conv_45, + num_block=blocks[3], + num_out=128, + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + num_group=256, + name="res_5") + conv_6_sep = Conv(conv_5, + num_filter=512, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + name="conv_6sep") fc1 = symbol_utils.get_fc1(conv_6_sep, num_classes, fc_type) return fc1 - diff --git a/recognition/symbol/fmobilenet.py b/recognition/symbol/fmobilenet.py index 1b6837a..bdbf8a5 100644 --- a/recognition/symbol/fmobilenet.py +++ b/recognition/symbol/fmobilenet.py @@ -22,66 +22,254 @@ import symbol_utils sys.path.append(os.path.join(os.path.dirname(__file__), '..')) from config import config + def Act(data, act_type, name): - #ignore param act_type, set it in this function - if act_type=='prelu': - body = mx.sym.LeakyReLU(data = data, act_type='prelu', name = name) + #ignore param act_type, set it in this function + if act_type == 'prelu': + body = mx.sym.LeakyReLU(data=data, act_type='prelu', name=name) else: - body = mx.sym.Activation(data=data, act_type=act_type, name=name) + body = mx.sym.Activation(data=data, act_type=act_type, name=name) return body -def Conv(data, num_filter=1, kernel=(1, 1), stride=(1, 1), pad=(0, 0), num_group=1, name=None, suffix=''): - conv = mx.sym.Convolution(data=data, num_filter=num_filter, kernel=kernel, num_group=num_group, stride=stride, pad=pad, no_bias=True, name='%s%s_conv2d' %(name, suffix)) - bn = mx.sym.BatchNorm(data=conv, name='%s%s_batchnorm' %(name, suffix), fix_gamma=True) - act = Act(data=bn, act_type=config.net_act, name='%s%s_relu' %(name, suffix)) + +def Conv(data, + num_filter=1, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + num_group=1, + name=None, + suffix=''): + conv = mx.sym.Convolution(data=data, + num_filter=num_filter, + kernel=kernel, + num_group=num_group, + stride=stride, + pad=pad, + no_bias=True, + name='%s%s_conv2d' % (name, suffix)) + bn = mx.sym.BatchNorm(data=conv, + name='%s%s_batchnorm' % (name, suffix), + fix_gamma=True) + act = Act(data=bn, + act_type=config.net_act, + name='%s%s_relu' % (name, suffix)) return act -def ConvOnly(data, num_filter=1, kernel=(1, 1), stride=(1, 1), pad=(0, 0), num_group=1, name=None, suffix=''): - conv = mx.sym.Convolution(data=data, num_filter=num_filter, kernel=kernel, num_group=num_group, stride=stride, pad=pad, no_bias=True, name='%s%s_conv2d' %(name, suffix)) + +def ConvOnly(data, + num_filter=1, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + num_group=1, + name=None, + suffix=''): + conv = mx.sym.Convolution(data=data, + num_filter=num_filter, + kernel=kernel, + num_group=num_group, + stride=stride, + pad=pad, + no_bias=True, + name='%s%s_conv2d' % (name, suffix)) return conv + def get_symbol(): num_classes = config.emb_size bn_mom = config.bn_mom workspace = config.workspace - data = mx.symbol.Variable(name="data") # 224 - data = data-127.5 - data = data*0.0078125 + data = mx.symbol.Variable(name="data") # 224 + data = data - 127.5 + data = data * 0.0078125 fc_type = config.net_output - bf = int(32*config.net_multiplier) - if config.net_input==0: - conv_1 = Conv(data, num_filter=bf, kernel=(3, 3), pad=(1, 1), stride=(2, 2), name="conv_1") # 224/112 + bf = int(32 * config.net_multiplier) + if config.net_input == 0: + conv_1 = Conv(data, + num_filter=bf, + kernel=(3, 3), + pad=(1, 1), + stride=(2, 2), + name="conv_1") # 224/112 else: - conv_1 = Conv(data, num_filter=bf, kernel=(3, 3), pad=(1, 1), stride=(1, 1), name="conv_1") # 224/112 - conv_2_dw = Conv(conv_1, num_group=bf, num_filter=bf, kernel=(3, 3), pad=(1, 1), stride=(1, 1), name="conv_2_dw") # 112/112 - conv_2 = Conv(conv_2_dw, num_filter=bf*2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_2") # 112/112 - conv_3_dw = Conv(conv_2, num_group=bf*2, num_filter=bf*2, kernel=(3, 3), pad=(1, 1), stride=(2, 2), name="conv_3_dw") # 112/56 - conv_3 = Conv(conv_3_dw, num_filter=bf*4, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_3") # 56/56 - conv_4_dw = Conv(conv_3, num_group=bf*4, num_filter=bf*4, kernel=(3, 3), pad=(1, 1), stride=(1, 1), name="conv_4_dw") # 56/56 - conv_4 = Conv(conv_4_dw, num_filter=bf*4, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_4") # 56/56 - conv_5_dw = Conv(conv_4, num_group=bf*4, num_filter=bf*4, kernel=(3, 3), pad=(1, 1), stride=(2, 2), name="conv_5_dw") # 56/28 - conv_5 = Conv(conv_5_dw, num_filter=bf*8, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_5") # 28/28 - conv_6_dw = Conv(conv_5, num_group=bf*8, num_filter=bf*8, kernel=(3, 3), pad=(1, 1), stride=(1, 1), name="conv_6_dw") # 28/28 - conv_6 = Conv(conv_6_dw, num_filter=bf*8, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_6") # 28/28 - conv_7_dw = Conv(conv_6, num_group=bf*8, num_filter=bf*8, kernel=(3, 3), pad=(1, 1), stride=(2, 2), name="conv_7_dw") # 28/14 - conv_7 = Conv(conv_7_dw, num_filter=bf*16, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_7") # 14/14 + conv_1 = Conv(data, + num_filter=bf, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + name="conv_1") # 224/112 + conv_2_dw = Conv(conv_1, + num_group=bf, + num_filter=bf, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + name="conv_2_dw") # 112/112 + conv_2 = Conv(conv_2_dw, + num_filter=bf * 2, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + name="conv_2") # 112/112 + conv_3_dw = Conv(conv_2, + num_group=bf * 2, + num_filter=bf * 2, + kernel=(3, 3), + pad=(1, 1), + stride=(2, 2), + name="conv_3_dw") # 112/56 + conv_3 = Conv(conv_3_dw, + num_filter=bf * 4, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + name="conv_3") # 56/56 + conv_4_dw = Conv(conv_3, + num_group=bf * 4, + num_filter=bf * 4, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + name="conv_4_dw") # 56/56 + conv_4 = Conv(conv_4_dw, + num_filter=bf * 4, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + name="conv_4") # 56/56 + conv_5_dw = Conv(conv_4, + num_group=bf * 4, + num_filter=bf * 4, + kernel=(3, 3), + pad=(1, 1), + stride=(2, 2), + name="conv_5_dw") # 56/28 + conv_5 = Conv(conv_5_dw, + num_filter=bf * 8, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + name="conv_5") # 28/28 + conv_6_dw = Conv(conv_5, + num_group=bf * 8, + num_filter=bf * 8, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + name="conv_6_dw") # 28/28 + conv_6 = Conv(conv_6_dw, + num_filter=bf * 8, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + name="conv_6") # 28/28 + conv_7_dw = Conv(conv_6, + num_group=bf * 8, + num_filter=bf * 8, + kernel=(3, 3), + pad=(1, 1), + stride=(2, 2), + name="conv_7_dw") # 28/14 + conv_7 = Conv(conv_7_dw, + num_filter=bf * 16, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + name="conv_7") # 14/14 - conv_8_dw = Conv(conv_7, num_group=bf*16, num_filter=bf*16, kernel=(3, 3), pad=(1, 1), stride=(1, 1), name="conv_8_dw") # 14/14 - conv_8 = Conv(conv_8_dw, num_filter=bf*16, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_8") # 14/14 - conv_9_dw = Conv(conv_8, num_group=bf*16, num_filter=bf*16, kernel=(3, 3), pad=(1, 1), stride=(1, 1), name="conv_9_dw") # 14/14 - conv_9 = Conv(conv_9_dw, num_filter=bf*16, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_9") # 14/14 - conv_10_dw = Conv(conv_9, num_group=bf*16, num_filter=bf*16, kernel=(3, 3), pad=(1, 1), stride=(1, 1), name="conv_10_dw") # 14/14 - conv_10 = Conv(conv_10_dw, num_filter=bf*16, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_10") # 14/14 - conv_11_dw = Conv(conv_10, num_group=bf*16, num_filter=bf*16, kernel=(3, 3), pad=(1, 1), stride=(1, 1), name="conv_11_dw") # 14/14 - conv_11 = Conv(conv_11_dw, num_filter=bf*16, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_11") # 14/14 - conv_12_dw = Conv(conv_11, num_group=bf*16, num_filter=bf*16, kernel=(3, 3), pad=(1, 1), stride=(1, 1), name="conv_12_dw") # 14/14 - conv_12 = Conv(conv_12_dw, num_filter=bf*16, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_12") # 14/14 + conv_8_dw = Conv(conv_7, + num_group=bf * 16, + num_filter=bf * 16, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + name="conv_8_dw") # 14/14 + conv_8 = Conv(conv_8_dw, + num_filter=bf * 16, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + name="conv_8") # 14/14 + conv_9_dw = Conv(conv_8, + num_group=bf * 16, + num_filter=bf * 16, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + name="conv_9_dw") # 14/14 + conv_9 = Conv(conv_9_dw, + num_filter=bf * 16, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + name="conv_9") # 14/14 + conv_10_dw = Conv(conv_9, + num_group=bf * 16, + num_filter=bf * 16, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + name="conv_10_dw") # 14/14 + conv_10 = Conv(conv_10_dw, + num_filter=bf * 16, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + name="conv_10") # 14/14 + conv_11_dw = Conv(conv_10, + num_group=bf * 16, + num_filter=bf * 16, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + name="conv_11_dw") # 14/14 + conv_11 = Conv(conv_11_dw, + num_filter=bf * 16, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + name="conv_11") # 14/14 + conv_12_dw = Conv(conv_11, + num_group=bf * 16, + num_filter=bf * 16, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + name="conv_12_dw") # 14/14 + conv_12 = Conv(conv_12_dw, + num_filter=bf * 16, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + name="conv_12") # 14/14 - conv_13_dw = Conv(conv_12, num_group=bf*16, num_filter=bf*16, kernel=(3, 3), pad=(1, 1), stride=(2, 2), name="conv_13_dw") # 14/7 - conv_13 = Conv(conv_13_dw, num_filter=bf*32, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_13") # 7/7 - conv_14_dw = Conv(conv_13, num_group=bf*32, num_filter=bf*32, kernel=(3, 3), pad=(1, 1), stride=(1, 1), name="conv_14_dw") # 7/7 - conv_14 = Conv(conv_14_dw, num_filter=bf*32, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_14") # 7/7 + conv_13_dw = Conv(conv_12, + num_group=bf * 16, + num_filter=bf * 16, + kernel=(3, 3), + pad=(1, 1), + stride=(2, 2), + name="conv_13_dw") # 14/7 + conv_13 = Conv(conv_13_dw, + num_filter=bf * 32, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + name="conv_13") # 7/7 + conv_14_dw = Conv(conv_13, + num_group=bf * 32, + num_filter=bf * 32, + kernel=(3, 3), + pad=(1, 1), + stride=(1, 1), + name="conv_14_dw") # 7/7 + conv_14 = Conv(conv_14_dw, + num_filter=bf * 32, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + name="conv_14") # 7/7 body = conv_14 fc1 = symbol_utils.get_fc1(body, num_classes, fc_type) return fc1 - diff --git a/recognition/symbol/fresnet.py b/recognition/symbol/fresnet.py index 0947385..b17920a 100644 --- a/recognition/symbol/fresnet.py +++ b/recognition/symbol/fresnet.py @@ -14,7 +14,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - ''' Adapted from https://github.com/tornadomeet/ResNet/blob/master/symbol_resnet.py Original author Wei Wu @@ -36,6 +35,7 @@ import sklearn sys.path.append(os.path.join(os.path.dirname(__file__), '..')) from config import config + def Conv(**kwargs): #name = kwargs.get('name') #_weight = mx.symbol.Variable(name+'_weight') @@ -46,13 +46,15 @@ def Conv(**kwargs): def Act(data, act_type, name): - if act_type=='prelu': - body = mx.sym.LeakyReLU(data = data, act_type='prelu', name = name) + if act_type == 'prelu': + body = mx.sym.LeakyReLU(data=data, act_type='prelu', name=name) else: - body = mx.symbol.Activation(data=data, act_type=act_type, name=name) + body = mx.symbol.Activation(data=data, act_type=act_type, name=name) return body -def residual_unit_v1(data, num_filter, stride, dim_match, name, bottle_neck, **kwargs): + +def residual_unit_v1(data, num_filter, stride, dim_match, name, bottle_neck, + **kwargs): """Return ResNet Unit symbol for building ResNet Parameters ---------- @@ -78,70 +80,176 @@ def residual_unit_v1(data, num_filter, stride, dim_match, name, bottle_neck, **k act_type = kwargs.get('version_act', 'prelu') #print('in unit1') if bottle_neck: - conv1 = Conv(data=data, num_filter=int(num_filter*0.25), kernel=(1,1), stride=stride, pad=(0,0), - no_bias=True, workspace=workspace, name=name + '_conv1') - bn1 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn1') + conv1 = Conv(data=data, + num_filter=int(num_filter * 0.25), + kernel=(1, 1), + stride=stride, + pad=(0, 0), + no_bias=True, + workspace=workspace, + name=name + '_conv1') + bn1 = mx.sym.BatchNorm(data=conv1, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn1') act1 = Act(data=bn1, act_type=act_type, name=name + '_relu1') - conv2 = Conv(data=act1, num_filter=int(num_filter*0.25), kernel=(3,3), stride=(1,1), pad=(1,1), - no_bias=True, workspace=workspace, name=name + '_conv2') - bn2 = mx.sym.BatchNorm(data=conv2, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn2') + conv2 = Conv(data=act1, + num_filter=int(num_filter * 0.25), + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv2') + bn2 = mx.sym.BatchNorm(data=conv2, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn2') act2 = Act(data=bn2, act_type=act_type, name=name + '_relu2') - conv3 = Conv(data=act2, num_filter=num_filter, kernel=(1,1), stride=(1,1), pad=(0,0), no_bias=True, - workspace=workspace, name=name + '_conv3') - bn3 = mx.sym.BatchNorm(data=conv3, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn3') + conv3 = Conv(data=act2, + num_filter=num_filter, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + no_bias=True, + workspace=workspace, + name=name + '_conv3') + bn3 = mx.sym.BatchNorm(data=conv3, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn3') if use_se: - #se begin - body = mx.sym.Pooling(data=bn3, global_pool=True, kernel=(7, 7), pool_type='avg', name=name+'_se_pool1') - body = Conv(data=body, num_filter=num_filter//16, kernel=(1,1), stride=(1,1), pad=(0,0), - name=name+"_se_conv1", workspace=workspace) - body = Act(data=body, act_type=act_type, name=name+'_se_relu1') - body = Conv(data=body, num_filter=num_filter, kernel=(1,1), stride=(1,1), pad=(0,0), - name=name+"_se_conv2", workspace=workspace) - body = mx.symbol.Activation(data=body, act_type='sigmoid', name=name+"_se_sigmoid") - bn3 = mx.symbol.broadcast_mul(bn3, body) - #se end + #se begin + body = mx.sym.Pooling(data=bn3, + global_pool=True, + kernel=(7, 7), + pool_type='avg', + name=name + '_se_pool1') + body = Conv(data=body, + num_filter=num_filter // 16, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv1", + workspace=workspace) + body = Act(data=body, act_type=act_type, name=name + '_se_relu1') + body = Conv(data=body, + num_filter=num_filter, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv2", + workspace=workspace) + body = mx.symbol.Activation(data=body, + act_type='sigmoid', + name=name + "_se_sigmoid") + bn3 = mx.symbol.broadcast_mul(bn3, body) + #se end if dim_match: shortcut = data else: - conv1sc = Conv(data=data, num_filter=num_filter, kernel=(1,1), stride=stride, no_bias=True, - workspace=workspace, name=name+'_conv1sc') - shortcut = mx.sym.BatchNorm(data=conv1sc, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_sc') + conv1sc = Conv(data=data, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + no_bias=True, + workspace=workspace, + name=name + '_conv1sc') + shortcut = mx.sym.BatchNorm(data=conv1sc, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_sc') if memonger: shortcut._set_attr(mirror_stage='True') - return Act(data=bn3 + shortcut, act_type=act_type, name=name + '_relu3') + return Act(data=bn3 + shortcut, + act_type=act_type, + name=name + '_relu3') else: - conv1 = Conv(data=data, num_filter=num_filter, kernel=(3,3), stride=stride, pad=(1,1), - no_bias=True, workspace=workspace, name=name + '_conv1') - bn1 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_bn1') + conv1 = Conv(data=data, + num_filter=num_filter, + kernel=(3, 3), + stride=stride, + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv1') + bn1 = mx.sym.BatchNorm(data=conv1, + fix_gamma=False, + momentum=bn_mom, + eps=2e-5, + name=name + '_bn1') act1 = Act(data=bn1, act_type=act_type, name=name + '_relu1') - conv2 = Conv(data=act1, num_filter=num_filter, kernel=(3,3), stride=(1,1), pad=(1,1), - no_bias=True, workspace=workspace, name=name + '_conv2') - bn2 = mx.sym.BatchNorm(data=conv2, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_bn2') + conv2 = Conv(data=act1, + num_filter=num_filter, + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv2') + bn2 = mx.sym.BatchNorm(data=conv2, + fix_gamma=False, + momentum=bn_mom, + eps=2e-5, + name=name + '_bn2') if use_se: - #se begin - body = mx.sym.Pooling(data=bn2, global_pool=True, kernel=(7, 7), pool_type='avg', name=name+'_se_pool1') - body = Conv(data=body, num_filter=num_filter//16, kernel=(1,1), stride=(1,1), pad=(0,0), - name=name+"_se_conv1", workspace=workspace) - body = Act(data=body, act_type=act_type, name=name+'_se_relu1') - body = Conv(data=body, num_filter=num_filter, kernel=(1,1), stride=(1,1), pad=(0,0), - name=name+"_se_conv2", workspace=workspace) - body = mx.symbol.Activation(data=body, act_type='sigmoid', name=name+"_se_sigmoid") - bn2 = mx.symbol.broadcast_mul(bn2, body) - #se end + #se begin + body = mx.sym.Pooling(data=bn2, + global_pool=True, + kernel=(7, 7), + pool_type='avg', + name=name + '_se_pool1') + body = Conv(data=body, + num_filter=num_filter // 16, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv1", + workspace=workspace) + body = Act(data=body, act_type=act_type, name=name + '_se_relu1') + body = Conv(data=body, + num_filter=num_filter, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv2", + workspace=workspace) + body = mx.symbol.Activation(data=body, + act_type='sigmoid', + name=name + "_se_sigmoid") + bn2 = mx.symbol.broadcast_mul(bn2, body) + #se end if dim_match: shortcut = data else: - conv1sc = Conv(data=data, num_filter=num_filter, kernel=(1,1), stride=stride, no_bias=True, - workspace=workspace, name=name+'_conv1sc') - shortcut = mx.sym.BatchNorm(data=conv1sc, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_sc') + conv1sc = Conv(data=data, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + no_bias=True, + workspace=workspace, + name=name + '_conv1sc') + shortcut = mx.sym.BatchNorm(data=conv1sc, + fix_gamma=False, + momentum=bn_mom, + eps=2e-5, + name=name + '_sc') if memonger: shortcut._set_attr(mirror_stage='True') - return Act(data=bn2 + shortcut, act_type=act_type, name=name + '_relu3') + return Act(data=bn2 + shortcut, + act_type=act_type, + name=name + '_relu3') -def residual_unit_v1_L(data, num_filter, stride, dim_match, name, bottle_neck, **kwargs): + +def residual_unit_v1_L(data, num_filter, stride, dim_match, name, bottle_neck, + **kwargs): """Return ResNet Unit symbol for building ResNet Parameters ---------- @@ -167,70 +275,176 @@ def residual_unit_v1_L(data, num_filter, stride, dim_match, name, bottle_neck, * act_type = kwargs.get('version_act', 'prelu') #print('in unit1') if bottle_neck: - conv1 = Conv(data=data, num_filter=int(num_filter*0.25), kernel=(1,1), stride=(1,1), pad=(0,0), - no_bias=True, workspace=workspace, name=name + '_conv1') - bn1 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn1') + conv1 = Conv(data=data, + num_filter=int(num_filter * 0.25), + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + no_bias=True, + workspace=workspace, + name=name + '_conv1') + bn1 = mx.sym.BatchNorm(data=conv1, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn1') act1 = Act(data=bn1, act_type=act_type, name=name + '_relu1') - conv2 = Conv(data=act1, num_filter=int(num_filter*0.25), kernel=(3,3), stride=(1,1), pad=(1,1), - no_bias=True, workspace=workspace, name=name + '_conv2') - bn2 = mx.sym.BatchNorm(data=conv2, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn2') + conv2 = Conv(data=act1, + num_filter=int(num_filter * 0.25), + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv2') + bn2 = mx.sym.BatchNorm(data=conv2, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn2') act2 = Act(data=bn2, act_type=act_type, name=name + '_relu2') - conv3 = Conv(data=act2, num_filter=num_filter, kernel=(1,1), stride=stride, pad=(0,0), no_bias=True, - workspace=workspace, name=name + '_conv3') - bn3 = mx.sym.BatchNorm(data=conv3, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn3') + conv3 = Conv(data=act2, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + pad=(0, 0), + no_bias=True, + workspace=workspace, + name=name + '_conv3') + bn3 = mx.sym.BatchNorm(data=conv3, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn3') if use_se: - #se begin - body = mx.sym.Pooling(data=bn3, global_pool=True, kernel=(7, 7), pool_type='avg', name=name+'_se_pool1') - body = Conv(data=body, num_filter=num_filter//16, kernel=(1,1), stride=(1,1), pad=(0,0), - name=name+"_se_conv1", workspace=workspace) - body = Act(data=body, act_type=act_type, name=name+'_se_relu1') - body = Conv(data=body, num_filter=num_filter, kernel=(1,1), stride=(1,1), pad=(0,0), - name=name+"_se_conv2", workspace=workspace) - body = mx.symbol.Activation(data=body, act_type='sigmoid', name=name+"_se_sigmoid") - bn3 = mx.symbol.broadcast_mul(bn3, body) - #se end + #se begin + body = mx.sym.Pooling(data=bn3, + global_pool=True, + kernel=(7, 7), + pool_type='avg', + name=name + '_se_pool1') + body = Conv(data=body, + num_filter=num_filter // 16, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv1", + workspace=workspace) + body = Act(data=body, act_type=act_type, name=name + '_se_relu1') + body = Conv(data=body, + num_filter=num_filter, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv2", + workspace=workspace) + body = mx.symbol.Activation(data=body, + act_type='sigmoid', + name=name + "_se_sigmoid") + bn3 = mx.symbol.broadcast_mul(bn3, body) + #se end if dim_match: shortcut = data else: - conv1sc = Conv(data=data, num_filter=num_filter, kernel=(1,1), stride=stride, no_bias=True, - workspace=workspace, name=name+'_conv1sc') - shortcut = mx.sym.BatchNorm(data=conv1sc, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_sc') + conv1sc = Conv(data=data, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + no_bias=True, + workspace=workspace, + name=name + '_conv1sc') + shortcut = mx.sym.BatchNorm(data=conv1sc, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_sc') if memonger: shortcut._set_attr(mirror_stage='True') - return Act(data=bn3 + shortcut, act_type=act_type, name=name + '_relu3') + return Act(data=bn3 + shortcut, + act_type=act_type, + name=name + '_relu3') else: - conv1 = Conv(data=data, num_filter=num_filter, kernel=(3,3), stride=(1,1), pad=(1,1), - no_bias=True, workspace=workspace, name=name + '_conv1') - bn1 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_bn1') + conv1 = Conv(data=data, + num_filter=num_filter, + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv1') + bn1 = mx.sym.BatchNorm(data=conv1, + fix_gamma=False, + momentum=bn_mom, + eps=2e-5, + name=name + '_bn1') act1 = Act(data=bn1, act_type=act_type, name=name + '_relu1') - conv2 = Conv(data=act1, num_filter=num_filter, kernel=(3,3), stride=stride, pad=(1,1), - no_bias=True, workspace=workspace, name=name + '_conv2') - bn2 = mx.sym.BatchNorm(data=conv2, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_bn2') + conv2 = Conv(data=act1, + num_filter=num_filter, + kernel=(3, 3), + stride=stride, + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv2') + bn2 = mx.sym.BatchNorm(data=conv2, + fix_gamma=False, + momentum=bn_mom, + eps=2e-5, + name=name + '_bn2') if use_se: - #se begin - body = mx.sym.Pooling(data=bn2, global_pool=True, kernel=(7, 7), pool_type='avg', name=name+'_se_pool1') - body = Conv(data=body, num_filter=num_filter//16, kernel=(1,1), stride=(1,1), pad=(0,0), - name=name+"_se_conv1", workspace=workspace) - body = Act(data=body, act_type=act_type, name=name+'_se_relu1') - body = Conv(data=body, num_filter=num_filter, kernel=(1,1), stride=(1,1), pad=(0,0), - name=name+"_se_conv2", workspace=workspace) - body = mx.symbol.Activation(data=body, act_type='sigmoid', name=name+"_se_sigmoid") - bn2 = mx.symbol.broadcast_mul(bn2, body) - #se end + #se begin + body = mx.sym.Pooling(data=bn2, + global_pool=True, + kernel=(7, 7), + pool_type='avg', + name=name + '_se_pool1') + body = Conv(data=body, + num_filter=num_filter // 16, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv1", + workspace=workspace) + body = Act(data=body, act_type=act_type, name=name + '_se_relu1') + body = Conv(data=body, + num_filter=num_filter, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv2", + workspace=workspace) + body = mx.symbol.Activation(data=body, + act_type='sigmoid', + name=name + "_se_sigmoid") + bn2 = mx.symbol.broadcast_mul(bn2, body) + #se end if dim_match: shortcut = data else: - conv1sc = Conv(data=data, num_filter=num_filter, kernel=(1,1), stride=stride, no_bias=True, - workspace=workspace, name=name+'_conv1sc') - shortcut = mx.sym.BatchNorm(data=conv1sc, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_sc') + conv1sc = Conv(data=data, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + no_bias=True, + workspace=workspace, + name=name + '_conv1sc') + shortcut = mx.sym.BatchNorm(data=conv1sc, + fix_gamma=False, + momentum=bn_mom, + eps=2e-5, + name=name + '_sc') if memonger: shortcut._set_attr(mirror_stage='True') - return Act(data=bn2 + shortcut, act_type=act_type, name=name + '_relu3') + return Act(data=bn2 + shortcut, + act_type=act_type, + name=name + '_relu3') -def residual_unit_v2(data, num_filter, stride, dim_match, name, bottle_neck, **kwargs): + +def residual_unit_v2(data, num_filter, stride, dim_match, name, bottle_neck, + **kwargs): """Return ResNet Unit symbol for building ResNet Parameters ---------- @@ -257,66 +471,159 @@ def residual_unit_v2(data, num_filter, stride, dim_match, name, bottle_neck, **k #print('in unit2') if bottle_neck: # the same as https://github.com/facebook/fb.resnet.torch#notes, a bit difference with origin paper - bn1 = mx.sym.BatchNorm(data=data, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn1') + bn1 = mx.sym.BatchNorm(data=data, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn1') act1 = Act(data=bn1, act_type=act_type, name=name + '_relu1') - conv1 = Conv(data=act1, num_filter=int(num_filter*0.25), kernel=(1,1), stride=(1,1), pad=(0,0), - no_bias=True, workspace=workspace, name=name + '_conv1') - bn2 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn2') + conv1 = Conv(data=act1, + num_filter=int(num_filter * 0.25), + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + no_bias=True, + workspace=workspace, + name=name + '_conv1') + bn2 = mx.sym.BatchNorm(data=conv1, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn2') act2 = Act(data=bn2, act_type=act_type, name=name + '_relu2') - conv2 = Conv(data=act2, num_filter=int(num_filter*0.25), kernel=(3,3), stride=stride, pad=(1,1), - no_bias=True, workspace=workspace, name=name + '_conv2') - bn3 = mx.sym.BatchNorm(data=conv2, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn3') + conv2 = Conv(data=act2, + num_filter=int(num_filter * 0.25), + kernel=(3, 3), + stride=stride, + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv2') + bn3 = mx.sym.BatchNorm(data=conv2, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn3') act3 = Act(data=bn3, act_type=act_type, name=name + '_relu3') - conv3 = Conv(data=act3, num_filter=num_filter, kernel=(1,1), stride=(1,1), pad=(0,0), no_bias=True, - workspace=workspace, name=name + '_conv3') + conv3 = Conv(data=act3, + num_filter=num_filter, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + no_bias=True, + workspace=workspace, + name=name + '_conv3') if use_se: - #se begin - body = mx.sym.Pooling(data=conv3, global_pool=True, kernel=(7, 7), pool_type='avg', name=name+'_se_pool1') - body = Conv(data=body, num_filter=num_filter//16, kernel=(1,1), stride=(1,1), pad=(0,0), - name=name+"_se_conv1", workspace=workspace) - body = Act(data=body, act_type=act_type, name=name+'_se_relu1') - body = Conv(data=body, num_filter=num_filter, kernel=(1,1), stride=(1,1), pad=(0,0), - name=name+"_se_conv2", workspace=workspace) - body = mx.symbol.Activation(data=body, act_type='sigmoid', name=name+"_se_sigmoid") - conv3 = mx.symbol.broadcast_mul(conv3, body) + #se begin + body = mx.sym.Pooling(data=conv3, + global_pool=True, + kernel=(7, 7), + pool_type='avg', + name=name + '_se_pool1') + body = Conv(data=body, + num_filter=num_filter // 16, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv1", + workspace=workspace) + body = Act(data=body, act_type=act_type, name=name + '_se_relu1') + body = Conv(data=body, + num_filter=num_filter, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv2", + workspace=workspace) + body = mx.symbol.Activation(data=body, + act_type='sigmoid', + name=name + "_se_sigmoid") + conv3 = mx.symbol.broadcast_mul(conv3, body) if dim_match: shortcut = data else: - shortcut = Conv(data=act1, num_filter=num_filter, kernel=(1,1), stride=stride, no_bias=True, - workspace=workspace, name=name+'_sc') + shortcut = Conv(data=act1, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + no_bias=True, + workspace=workspace, + name=name + '_sc') if memonger: shortcut._set_attr(mirror_stage='True') return conv3 + shortcut else: - bn1 = mx.sym.BatchNorm(data=data, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_bn1') + bn1 = mx.sym.BatchNorm(data=data, + fix_gamma=False, + momentum=bn_mom, + eps=2e-5, + name=name + '_bn1') act1 = Act(data=bn1, act_type=act_type, name=name + '_relu1') - conv1 = Conv(data=act1, num_filter=num_filter, kernel=(3,3), stride=stride, pad=(1,1), - no_bias=True, workspace=workspace, name=name + '_conv1') - bn2 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_bn2') + conv1 = Conv(data=act1, + num_filter=num_filter, + kernel=(3, 3), + stride=stride, + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv1') + bn2 = mx.sym.BatchNorm(data=conv1, + fix_gamma=False, + momentum=bn_mom, + eps=2e-5, + name=name + '_bn2') act2 = Act(data=bn2, act_type=act_type, name=name + '_relu2') - conv2 = Conv(data=act2, num_filter=num_filter, kernel=(3,3), stride=(1,1), pad=(1,1), - no_bias=True, workspace=workspace, name=name + '_conv2') + conv2 = Conv(data=act2, + num_filter=num_filter, + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv2') if use_se: - #se begin - body = mx.sym.Pooling(data=conv2, global_pool=True, kernel=(7, 7), pool_type='avg', name=name+'_se_pool1') - body = Conv(data=body, num_filter=num_filter//16, kernel=(1,1), stride=(1,1), pad=(0,0), - name=name+"_se_conv1", workspace=workspace) - body = Act(data=body, act_type=act_type, name=name+'_se_relu1') - body = Conv(data=body, num_filter=num_filter, kernel=(1,1), stride=(1,1), pad=(0,0), - name=name+"_se_conv2", workspace=workspace) - body = mx.symbol.Activation(data=body, act_type='sigmoid', name=name+"_se_sigmoid") - conv2 = mx.symbol.broadcast_mul(conv2, body) + #se begin + body = mx.sym.Pooling(data=conv2, + global_pool=True, + kernel=(7, 7), + pool_type='avg', + name=name + '_se_pool1') + body = Conv(data=body, + num_filter=num_filter // 16, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv1", + workspace=workspace) + body = Act(data=body, act_type=act_type, name=name + '_se_relu1') + body = Conv(data=body, + num_filter=num_filter, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv2", + workspace=workspace) + body = mx.symbol.Activation(data=body, + act_type='sigmoid', + name=name + "_se_sigmoid") + conv2 = mx.symbol.broadcast_mul(conv2, body) if dim_match: shortcut = data else: - shortcut = Conv(data=act1, num_filter=num_filter, kernel=(1,1), stride=stride, no_bias=True, - workspace=workspace, name=name+'_sc') + shortcut = Conv(data=act1, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + no_bias=True, + workspace=workspace, + name=name + '_sc') if memonger: shortcut._set_attr(mirror_stage='True') return conv2 + shortcut -def residual_unit_v3(data, num_filter, stride, dim_match, name, bottle_neck, **kwargs): - + +def residual_unit_v3(data, num_filter, stride, dim_match, name, bottle_neck, + **kwargs): """Return ResNet Unit symbol for building ResNet Parameters ---------- @@ -342,73 +649,182 @@ def residual_unit_v3(data, num_filter, stride, dim_match, name, bottle_neck, **k act_type = kwargs.get('version_act', 'prelu') #print('in unit3') if bottle_neck: - bn1 = mx.sym.BatchNorm(data=data, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn1') - conv1 = Conv(data=bn1, num_filter=int(num_filter*0.25), kernel=(1,1), stride=(1,1), pad=(0,0), - no_bias=True, workspace=workspace, name=name + '_conv1') - bn2 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn2') + bn1 = mx.sym.BatchNorm(data=data, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn1') + conv1 = Conv(data=bn1, + num_filter=int(num_filter * 0.25), + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + no_bias=True, + workspace=workspace, + name=name + '_conv1') + bn2 = mx.sym.BatchNorm(data=conv1, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn2') act1 = Act(data=bn2, act_type=act_type, name=name + '_relu1') - conv2 = Conv(data=act1, num_filter=int(num_filter*0.25), kernel=(3,3), stride=(1,1), pad=(1,1), - no_bias=True, workspace=workspace, name=name + '_conv2') - bn3 = mx.sym.BatchNorm(data=conv2, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn3') + conv2 = Conv(data=act1, + num_filter=int(num_filter * 0.25), + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv2') + bn3 = mx.sym.BatchNorm(data=conv2, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn3') act2 = Act(data=bn3, act_type=act_type, name=name + '_relu2') - conv3 = Conv(data=act2, num_filter=num_filter, kernel=(1,1), stride=stride, pad=(0,0), no_bias=True, - workspace=workspace, name=name + '_conv3') - bn4 = mx.sym.BatchNorm(data=conv3, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn4') + conv3 = Conv(data=act2, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + pad=(0, 0), + no_bias=True, + workspace=workspace, + name=name + '_conv3') + bn4 = mx.sym.BatchNorm(data=conv3, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn4') if use_se: - #se begin - body = mx.sym.Pooling(data=bn4, global_pool=True, kernel=(7, 7), pool_type='avg', name=name+'_se_pool1') - body = Conv(data=body, num_filter=num_filter//16, kernel=(1,1), stride=(1,1), pad=(0,0), - name=name+"_se_conv1", workspace=workspace) - body = Act(data=body, act_type=act_type, name=name+'_se_relu1') - body = Conv(data=body, num_filter=num_filter, kernel=(1,1), stride=(1,1), pad=(0,0), - name=name+"_se_conv2", workspace=workspace) - body = mx.symbol.Activation(data=body, act_type='sigmoid', name=name+"_se_sigmoid") - bn4 = mx.symbol.broadcast_mul(bn4, body) - #se end + #se begin + body = mx.sym.Pooling(data=bn4, + global_pool=True, + kernel=(7, 7), + pool_type='avg', + name=name + '_se_pool1') + body = Conv(data=body, + num_filter=num_filter // 16, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv1", + workspace=workspace) + body = Act(data=body, act_type=act_type, name=name + '_se_relu1') + body = Conv(data=body, + num_filter=num_filter, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv2", + workspace=workspace) + body = mx.symbol.Activation(data=body, + act_type='sigmoid', + name=name + "_se_sigmoid") + bn4 = mx.symbol.broadcast_mul(bn4, body) + #se end if dim_match: shortcut = data else: - conv1sc = Conv(data=data, num_filter=num_filter, kernel=(1,1), stride=stride, no_bias=True, - workspace=workspace, name=name+'_conv1sc') - shortcut = mx.sym.BatchNorm(data=conv1sc, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_sc') + conv1sc = Conv(data=data, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + no_bias=True, + workspace=workspace, + name=name + '_conv1sc') + shortcut = mx.sym.BatchNorm(data=conv1sc, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_sc') if memonger: shortcut._set_attr(mirror_stage='True') return bn4 + shortcut else: - bn1 = mx.sym.BatchNorm(data=data, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn1') - conv1 = Conv(data=bn1, num_filter=num_filter, kernel=(3,3), stride=(1,1), pad=(1,1), - no_bias=True, workspace=workspace, name=name + '_conv1') - bn2 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn2') + bn1 = mx.sym.BatchNorm(data=data, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn1') + conv1 = Conv(data=bn1, + num_filter=num_filter, + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv1') + bn2 = mx.sym.BatchNorm(data=conv1, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn2') act1 = Act(data=bn2, act_type=act_type, name=name + '_relu1') - conv2 = Conv(data=act1, num_filter=num_filter, kernel=(3,3), stride=stride, pad=(1,1), - no_bias=True, workspace=workspace, name=name + '_conv2') - bn3 = mx.sym.BatchNorm(data=conv2, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn3') + conv2 = Conv(data=act1, + num_filter=num_filter, + kernel=(3, 3), + stride=stride, + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv2') + bn3 = mx.sym.BatchNorm(data=conv2, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn3') if use_se: - #se begin - body = mx.sym.Pooling(data=bn3, global_pool=True, kernel=(7, 7), pool_type='avg', name=name+'_se_pool1') - body = Conv(data=body, num_filter=num_filter//16, kernel=(1,1), stride=(1,1), pad=(0,0), - name=name+"_se_conv1", workspace=workspace) - body = Act(data=body, act_type=act_type, name=name+'_se_relu1') - body = Conv(data=body, num_filter=num_filter, kernel=(1,1), stride=(1,1), pad=(0,0), - name=name+"_se_conv2", workspace=workspace) - body = mx.symbol.Activation(data=body, act_type='sigmoid', name=name+"_se_sigmoid") - bn3 = mx.symbol.broadcast_mul(bn3, body) - #se end + #se begin + body = mx.sym.Pooling(data=bn3, + global_pool=True, + kernel=(7, 7), + pool_type='avg', + name=name + '_se_pool1') + body = Conv(data=body, + num_filter=num_filter // 16, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv1", + workspace=workspace) + body = Act(data=body, act_type=act_type, name=name + '_se_relu1') + body = Conv(data=body, + num_filter=num_filter, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv2", + workspace=workspace) + body = mx.symbol.Activation(data=body, + act_type='sigmoid', + name=name + "_se_sigmoid") + bn3 = mx.symbol.broadcast_mul(bn3, body) + #se end if dim_match: shortcut = data else: - conv1sc = Conv(data=data, num_filter=num_filter, kernel=(1,1), stride=stride, no_bias=True, - workspace=workspace, name=name+'_conv1sc') - shortcut = mx.sym.BatchNorm(data=conv1sc, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_sc') + conv1sc = Conv(data=data, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + no_bias=True, + workspace=workspace, + name=name + '_conv1sc') + shortcut = mx.sym.BatchNorm(data=conv1sc, + fix_gamma=False, + momentum=bn_mom, + eps=2e-5, + name=name + '_sc') if memonger: shortcut._set_attr(mirror_stage='True') return bn3 + shortcut -def residual_unit_v3_x(data, num_filter, stride, dim_match, name, bottle_neck, **kwargs): - + +def residual_unit_v3_x(data, num_filter, stride, dim_match, name, bottle_neck, + **kwargs): """Return ResNeXt Unit symbol for building ResNeXt Parameters ---------- @@ -427,7 +843,7 @@ def residual_unit_v3_x(data, num_filter, stride, dim_match, name, bottle_neck, * workspace : int Workspace used in convolution operator """ - assert(bottle_neck) + assert (bottle_neck) use_se = kwargs.get('version_se', 1) bn_mom = kwargs.get('bn_mom', 0.9) workspace = kwargs.get('workspace', 256) @@ -435,61 +851,130 @@ def residual_unit_v3_x(data, num_filter, stride, dim_match, name, bottle_neck, * act_type = kwargs.get('version_act', 'prelu') num_group = 32 #print('in unit3') - bn1 = mx.sym.BatchNorm(data=data, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn1') - conv1 = Conv(data=bn1, num_group=num_group, num_filter=int(num_filter*0.5), kernel=(1,1), stride=(1,1), pad=(0,0), - no_bias=True, workspace=workspace, name=name + '_conv1') - bn2 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn2') + bn1 = mx.sym.BatchNorm(data=data, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn1') + conv1 = Conv(data=bn1, + num_group=num_group, + num_filter=int(num_filter * 0.5), + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + no_bias=True, + workspace=workspace, + name=name + '_conv1') + bn2 = mx.sym.BatchNorm(data=conv1, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn2') act1 = Act(data=bn2, act_type=act_type, name=name + '_relu1') - conv2 = Conv(data=act1, num_group=num_group, num_filter=int(num_filter*0.5), kernel=(3,3), stride=(1,1), pad=(1,1), - no_bias=True, workspace=workspace, name=name + '_conv2') - bn3 = mx.sym.BatchNorm(data=conv2, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn3') + conv2 = Conv(data=act1, + num_group=num_group, + num_filter=int(num_filter * 0.5), + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + workspace=workspace, + name=name + '_conv2') + bn3 = mx.sym.BatchNorm(data=conv2, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn3') act2 = Act(data=bn3, act_type=act_type, name=name + '_relu2') - conv3 = Conv(data=act2, num_filter=num_filter, kernel=(1,1), stride=stride, pad=(0,0), no_bias=True, - workspace=workspace, name=name + '_conv3') - bn4 = mx.sym.BatchNorm(data=conv3, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn4') + conv3 = Conv(data=act2, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + pad=(0, 0), + no_bias=True, + workspace=workspace, + name=name + '_conv3') + bn4 = mx.sym.BatchNorm(data=conv3, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_bn4') if use_se: - #se begin - body = mx.sym.Pooling(data=bn4, global_pool=True, kernel=(7, 7), pool_type='avg', name=name+'_se_pool1') - body = Conv(data=body, num_filter=num_filter//16, kernel=(1,1), stride=(1,1), pad=(0,0), - name=name+"_se_conv1", workspace=workspace) - body = Act(data=body, act_type=act_type, name=name+'_se_relu1') - body = Conv(data=body, num_filter=num_filter, kernel=(1,1), stride=(1,1), pad=(0,0), - name=name+"_se_conv2", workspace=workspace) - body = mx.symbol.Activation(data=body, act_type='sigmoid', name=name+"_se_sigmoid") - bn4 = mx.symbol.broadcast_mul(bn4, body) - #se end + #se begin + body = mx.sym.Pooling(data=bn4, + global_pool=True, + kernel=(7, 7), + pool_type='avg', + name=name + '_se_pool1') + body = Conv(data=body, + num_filter=num_filter // 16, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv1", + workspace=workspace) + body = Act(data=body, act_type=act_type, name=name + '_se_relu1') + body = Conv(data=body, + num_filter=num_filter, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + name=name + "_se_conv2", + workspace=workspace) + body = mx.symbol.Activation(data=body, + act_type='sigmoid', + name=name + "_se_sigmoid") + bn4 = mx.symbol.broadcast_mul(bn4, body) + #se end if dim_match: shortcut = data else: - conv1sc = Conv(data=data, num_filter=num_filter, kernel=(1,1), stride=stride, no_bias=True, - workspace=workspace, name=name+'_conv1sc') - shortcut = mx.sym.BatchNorm(data=conv1sc, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_sc') + conv1sc = Conv(data=data, + num_filter=num_filter, + kernel=(1, 1), + stride=stride, + no_bias=True, + workspace=workspace, + name=name + '_conv1sc') + shortcut = mx.sym.BatchNorm(data=conv1sc, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name=name + '_sc') if memonger: shortcut._set_attr(mirror_stage='True') return bn4 + shortcut -def residual_unit(data, num_filter, stride, dim_match, name, bottle_neck, **kwargs): - uv = kwargs.get('version_unit', 3) - version_input = kwargs.get('version_input', 1) - if uv==1: - if version_input==0: - return residual_unit_v1(data, num_filter, stride, dim_match, name, bottle_neck, **kwargs) +def residual_unit(data, num_filter, stride, dim_match, name, bottle_neck, + **kwargs): + uv = kwargs.get('version_unit', 3) + version_input = kwargs.get('version_input', 1) + if uv == 1: + if version_input == 0: + return residual_unit_v1(data, num_filter, stride, dim_match, name, + bottle_neck, **kwargs) + else: + return residual_unit_v1_L(data, num_filter, stride, dim_match, + name, bottle_neck, **kwargs) + elif uv == 2: + return residual_unit_v2(data, num_filter, stride, dim_match, name, + bottle_neck, **kwargs) + elif uv == 4: + return residual_unit_v4(data, num_filter, stride, dim_match, name, + bottle_neck, **kwargs) else: - return residual_unit_v1_L(data, num_filter, stride, dim_match, name, bottle_neck, **kwargs) - elif uv==2: - return residual_unit_v2(data, num_filter, stride, dim_match, name, bottle_neck, **kwargs) - elif uv==4: - return residual_unit_v4(data, num_filter, stride, dim_match, name, bottle_neck, **kwargs) - else: - return residual_unit_v3(data, num_filter, stride, dim_match, name, bottle_neck, **kwargs) + return residual_unit_v3(data, num_filter, stride, dim_match, name, + bottle_neck, **kwargs) + def resnet(units, num_stages, filter_list, num_classes, bottle_neck): bn_mom = config.bn_mom workspace = config.workspace - kwargs = {'version_se' : config.net_se, + kwargs = { + 'version_se': config.net_se, 'version_input': config.net_input, 'version_output': config.net_output, 'version_unit': config.net_unit, @@ -497,7 +982,7 @@ def resnet(units, num_stages, filter_list, num_classes, bottle_neck): 'bn_mom': bn_mom, 'workspace': workspace, 'memonger': config.memonger, - } + } """Return ResNet symbol of Parameters ---------- @@ -516,64 +1001,118 @@ def resnet(units, num_stages, filter_list, num_classes, bottle_neck): """ version_se = kwargs.get('version_se', 1) version_input = kwargs.get('version_input', 1) - assert version_input>=0 + assert version_input >= 0 version_output = kwargs.get('version_output', 'E') fc_type = version_output version_unit = kwargs.get('version_unit', 3) act_type = kwargs.get('version_act', 'prelu') memonger = kwargs.get('memonger', False) - print(version_se, version_input, version_output, version_unit, act_type, memonger) + print(version_se, version_input, version_output, version_unit, act_type, + memonger) num_unit = len(units) - assert(num_unit == num_stages) + assert (num_unit == num_stages) data = mx.sym.Variable(name='data') - if version_input==0: - #data = mx.sym.BatchNorm(data=data, fix_gamma=True, eps=2e-5, momentum=bn_mom, name='bn_data') - data = mx.sym.identity(data=data, name='id') - data = data-127.5 - data = data*0.0078125 - body = Conv(data=data, num_filter=filter_list[0], kernel=(7, 7), stride=(2,2), pad=(3, 3), - no_bias=True, name="conv0", workspace=workspace) - body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn0') - body = Act(data=body, act_type=act_type, name='relu0') - #body = mx.sym.Pooling(data=body, kernel=(3, 3), stride=(2,2), pad=(1,1), pool_type='max') - elif version_input==2: - data = mx.sym.BatchNorm(data=data, fix_gamma=True, eps=2e-5, momentum=bn_mom, name='bn_data') - body = Conv(data=data, num_filter=filter_list[0], kernel=(3,3), stride=(1,1), pad=(1,1), - no_bias=True, name="conv0", workspace=workspace) - body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn0') - body = Act(data=body, act_type=act_type, name='relu0') + if version_input == 0: + #data = mx.sym.BatchNorm(data=data, fix_gamma=True, eps=2e-5, momentum=bn_mom, name='bn_data') + data = mx.sym.identity(data=data, name='id') + data = data - 127.5 + data = data * 0.0078125 + body = Conv(data=data, + num_filter=filter_list[0], + kernel=(7, 7), + stride=(2, 2), + pad=(3, 3), + no_bias=True, + name="conv0", + workspace=workspace) + body = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name='bn0') + body = Act(data=body, act_type=act_type, name='relu0') + #body = mx.sym.Pooling(data=body, kernel=(3, 3), stride=(2,2), pad=(1,1), pool_type='max') + elif version_input == 2: + data = mx.sym.BatchNorm(data=data, + fix_gamma=True, + eps=2e-5, + momentum=bn_mom, + name='bn_data') + body = Conv(data=data, + num_filter=filter_list[0], + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + name="conv0", + workspace=workspace) + body = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name='bn0') + body = Act(data=body, act_type=act_type, name='relu0') else: - data = mx.sym.identity(data=data, name='id') - data = data-127.5 - data = data*0.0078125 - body = data - body = Conv(data=body, num_filter=filter_list[0], kernel=(3,3), stride=(1,1), pad=(1, 1), - no_bias=True, name="conv0", workspace=workspace) - body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn0') - body = Act(data=body, act_type=act_type, name='relu0') + data = mx.sym.identity(data=data, name='id') + data = data - 127.5 + data = data * 0.0078125 + body = data + body = Conv(data=body, + num_filter=filter_list[0], + kernel=(3, 3), + stride=(1, 1), + pad=(1, 1), + no_bias=True, + name="conv0", + workspace=workspace) + body = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name='bn0') + body = Act(data=body, act_type=act_type, name='relu0') for i in range(num_stages): - #if version_input==0: - # body = residual_unit(body, filter_list[i+1], (1 if i==0 else 2, 1 if i==0 else 2), False, - # name='stage%d_unit%d' % (i + 1, 1), bottle_neck=bottle_neck, **kwargs) - #else: - # body = residual_unit(body, filter_list[i+1], (2, 2), False, - # name='stage%d_unit%d' % (i + 1, 1), bottle_neck=bottle_neck, **kwargs) - body = residual_unit(body, filter_list[i+1], (2, 2), False, - name='stage%d_unit%d' % (i + 1, 1), bottle_neck=bottle_neck, **kwargs) - for j in range(units[i]-1): - body = residual_unit(body, filter_list[i+1], (1,1), True, name='stage%d_unit%d' % (i+1, j+2), - bottle_neck=bottle_neck, **kwargs) + #if version_input==0: + # body = residual_unit(body, filter_list[i+1], (1 if i==0 else 2, 1 if i==0 else 2), False, + # name='stage%d_unit%d' % (i + 1, 1), bottle_neck=bottle_neck, **kwargs) + #else: + # body = residual_unit(body, filter_list[i+1], (2, 2), False, + # name='stage%d_unit%d' % (i + 1, 1), bottle_neck=bottle_neck, **kwargs) + body = residual_unit(body, + filter_list[i + 1], (2, 2), + False, + name='stage%d_unit%d' % (i + 1, 1), + bottle_neck=bottle_neck, + **kwargs) + for j in range(units[i] - 1): + body = residual_unit(body, + filter_list[i + 1], (1, 1), + True, + name='stage%d_unit%d' % (i + 1, j + 2), + bottle_neck=bottle_neck, + **kwargs) if bottle_neck: - body = Conv(data=body, num_filter=512, kernel=(1,1), stride=(1,1), pad=(0,0), - no_bias=True, name="convd", workspace=workspace) - body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bnd') - body = Act(data=body, act_type=act_type, name='relud') + body = Conv(data=body, + num_filter=512, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + no_bias=True, + name="convd", + workspace=workspace) + body = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name='bnd') + body = Act(data=body, act_type=act_type, name='relud') fc1 = symbol_utils.get_fc1(body, num_classes, fc_type) return fc1 + def get_symbol(): """ Adapted from https://github.com/tornadomeet/ResNet/blob/master/train_resnet.py @@ -625,24 +1164,24 @@ def get_symbol(): elif num_layers == 269: units = [3, 30, 48, 8] else: - raise ValueError("no experiments done on num_layers {}, you can do it yourself".format(num_layers)) + raise ValueError( + "no experiments done on num_layers {}, you can do it yourself". + format(num_layers)) - net = resnet(units = units, - num_stages = num_stages, - filter_list = filter_list, - num_classes = num_classes, - bottle_neck = bottle_neck) + net = resnet(units=units, + num_stages=num_stages, + filter_list=filter_list, + num_classes=num_classes, + bottle_neck=bottle_neck) if config.memonger: - dshape = (config.per_batch_size, config.image_shape[2], config.image_shape[0], config.image_shape[1]) - net_mem_planned = memonger.search_plan(net, data=dshape) - old_cost = memonger.get_cost(net, data=dshape) - new_cost = memonger.get_cost(net_mem_planned, data=dshape) + dshape = (config.per_batch_size, config.image_shape[2], + config.image_shape[0], config.image_shape[1]) + net_mem_planned = memonger.search_plan(net, data=dshape) + old_cost = memonger.get_cost(net, data=dshape) + new_cost = memonger.get_cost(net_mem_planned, data=dshape) - print('Old feature map cost=%d MB' % old_cost) - print('New feature map cost=%d MB' % new_cost) - net = net_mem_planned + print('Old feature map cost=%d MB' % old_cost) + print('New feature map cost=%d MB' % new_cost) + net = net_mem_planned return net - - - diff --git a/recognition/symbol/memonger.py b/recognition/symbol/memonger.py index d054bda..8ad610b 100644 --- a/recognition/symbol/memonger.py +++ b/recognition/symbol/memonger.py @@ -1,6 +1,7 @@ import mxnet as mx import math + def prod(shape): """Get product of the shape. """ @@ -138,7 +139,10 @@ def search_plan(sym, ntrial=6, type_dict=None, **kwargs): for k in range(nbegin): info = {} - sym = make_mirror_plan(sym, threshold=threshold, plan_info=info, **kwargs) + sym = make_mirror_plan(sym, + threshold=threshold, + plan_info=info, + **kwargs) cost = get_cost(sym, type_dict, **kwargs) save_size = info['save_size'] >> 20 local_size = info['max_size'] >> 20 @@ -147,7 +151,7 @@ def search_plan(sym, ntrial=6, type_dict=None, **kwargs): min_cost = cost if min_threshold is None or local_size < min_threshold: min_threshold = local_size - print ("Search threshold=%d MB, cost=%d MB" % (threshold, cost)) + print("Search threshold=%d MB, cost=%d MB" % (threshold, cost)) history.append((cost, threshold, sym)) threshold = guess @@ -156,13 +160,16 @@ def search_plan(sym, ntrial=6, type_dict=None, **kwargs): threshold = min_threshold + step if step > 0: for k in range(ntrial): - sym = make_mirror_plan(sym, threshold=threshold, plan_info=info, **kwargs) + sym = make_mirror_plan(sym, + threshold=threshold, + plan_info=info, + **kwargs) cost = get_cost(sym, type_dict, **kwargs) - print ("Search threshold=%d MB, cost=%d MB" % (threshold, cost)) + print("Search threshold=%d MB, cost=%d MB" % (threshold, cost)) history.append((cost, threshold, sym)) threshold += step - history.sort(key = lambda x: x[0]) + history.sort(key=lambda x: x[0]) cost, threshold, sym = history[0] print('Find best plan with threshold=%d, cost=%d MB' % (threshold, cost)) return sym diff --git a/recognition/symbol/memonger_v2.py b/recognition/symbol/memonger_v2.py index 70f7a00..92963de 100644 --- a/recognition/symbol/memonger_v2.py +++ b/recognition/symbol/memonger_v2.py @@ -1,6 +1,7 @@ import mxnet as mx import math + def prod(shape): """Get product of the shape. """ @@ -138,7 +139,10 @@ def search_plan(sym, ntrial=6, type_dict=None, **kwargs): for k in range(nbegin): info = {} - sym = make_mirror_plan(sym, threshold=threshold, plan_info=info, **kwargs) + sym = make_mirror_plan(sym, + threshold=threshold, + plan_info=info, + **kwargs) cost = get_cost(sym, type_dict, **kwargs) save_size = info['save_size'] >> 20 local_size = info['max_size'] >> 20 @@ -147,7 +151,7 @@ def search_plan(sym, ntrial=6, type_dict=None, **kwargs): min_cost = cost if min_threshold is None or local_size < min_threshold: min_threshold = local_size - print ("Search threshold=%d MB, cost=%d MB" % (threshold, cost)) + print("Search threshold=%d MB, cost=%d MB" % (threshold, cost)) history.append((cost, threshold, sym)) threshold = guess @@ -156,19 +160,26 @@ def search_plan(sym, ntrial=6, type_dict=None, **kwargs): threshold = min_threshold + step if step > 0: for k in range(ntrial): - sym = make_mirror_plan(sym, threshold=threshold, plan_info=info, **kwargs) + sym = make_mirror_plan(sym, + threshold=threshold, + plan_info=info, + **kwargs) cost = get_cost(sym, type_dict, **kwargs) - print ("Search threshold=%d MB, cost=%d MB" % (threshold, cost)) + print("Search threshold=%d MB, cost=%d MB" % (threshold, cost)) history.append((cost, threshold, sym)) threshold += step - history.sort(key = lambda x: x[0]) + history.sort(key=lambda x: x[0]) cost, threshold, sym = history[0] print('Find best plan with threshold=%d, cost=%d MB' % (threshold, cost)) return sym -def make_mirror_plan_to_layer(sym,layer_name,threshold, plan_info=None, **kwargs): +def make_mirror_plan_to_layer(sym, + layer_name, + threshold, + plan_info=None, + **kwargs): """ sym is the original symbal layer_name is a name to which layer of the network should be set as mirror @@ -186,8 +197,8 @@ def make_mirror_plan_to_layer(sym,layer_name,threshold, plan_info=None, **kwargs max_size = 0 last_stage = '' stage_decision = '' - switch=True - + switch = True + for idx, item in enumerate(shape_dict): sb = internals[idx] name, shape = item @@ -200,7 +211,7 @@ def make_mirror_plan_to_layer(sym,layer_name,threshold, plan_info=None, **kwargs local_size += prod(shape) * 4 sb._set_attr(force_mirroring='True') print('set force_mirroring', name, total_size, local_size) - if layer_name!='' and layer_name in name: + if layer_name != '' and layer_name in name: switch = False if sb.attr('mirror_stage') is not None: @@ -226,7 +237,13 @@ def make_mirror_plan_to_layer(sym,layer_name,threshold, plan_info=None, **kwargs plan_info['save_size'] = save_size return sym -def search_plan_to_layer(sym,layer_name=None,threshold=500, ntrial=6, type_dict=None, **kwargs): + +def search_plan_to_layer(sym, + layer_name=None, + threshold=500, + ntrial=6, + type_dict=None, + **kwargs): """Quickly heurestic search over possible plans to find good memory plan. Parameters @@ -244,7 +261,11 @@ def search_plan_to_layer(sym,layer_name=None,threshold=500, ntrial=6, type_dict= for k in range(nbegin): info = {} - sym = make_mirror_plan_to_layer(sym,layer_name=layer_name,threshold=threshold, plan_info=info, **kwargs) + sym = make_mirror_plan_to_layer(sym, + layer_name=layer_name, + threshold=threshold, + plan_info=info, + **kwargs) cost = get_cost(sym, type_dict, **kwargs) save_size = info['save_size'] >> 20 local_size = info['max_size'] >> 20 @@ -253,7 +274,7 @@ def search_plan_to_layer(sym,layer_name=None,threshold=500, ntrial=6, type_dict= min_cost = cost if min_threshold is None or local_size < min_threshold: min_threshold = local_size - print ("Search threshold=%d MB, cost=%d MB" % (threshold, cost)) + print("Search threshold=%d MB, cost=%d MB" % (threshold, cost)) history.append((cost, threshold, sym)) threshold = guess @@ -263,13 +284,17 @@ def search_plan_to_layer(sym,layer_name=None,threshold=500, ntrial=6, type_dict= threshold = min_threshold + step if step > 0: for k in range(ntrial): - sym = make_mirror_plan_to_layer(sym,layer_name=layer_name, threshold=threshold, plan_info=info, **kwargs) + sym = make_mirror_plan_to_layer(sym, + layer_name=layer_name, + threshold=threshold, + plan_info=info, + **kwargs) cost = get_cost(sym, type_dict, **kwargs) - print ("Search threshold=%d MB, cost=%d MB" % (threshold, cost)) + print("Search threshold=%d MB, cost=%d MB" % (threshold, cost)) history.append((cost, threshold, sym)) threshold += step - history.sort(key = lambda x: x[0]) + history.sort(key=lambda x: x[0]) cost, threshold, sym = history[0] print('Find best plan with threshold=%d, cost=%d MB' % (threshold, cost)) return sym diff --git a/recognition/symbol/symbol_utils.py b/recognition/symbol/symbol_utils.py index 1bf22d7..3eb6f1c 100644 --- a/recognition/symbol/symbol_utils.py +++ b/recognition/symbol/symbol_utils.py @@ -4,6 +4,7 @@ import mxnet as mx sys.path.append(os.path.join(os.path.dirname(__file__), '..')) from config import config + def Conv(**kwargs): #name = kwargs.get('name') #_weight = mx.symbol.Variable(name+'_weight') @@ -12,107 +13,266 @@ def Conv(**kwargs): body = mx.sym.Convolution(**kwargs) return body + def Act(data, act_type, name): - #ignore param act_type, set it in this function - if act_type=='prelu': - body = mx.sym.LeakyReLU(data = data, act_type='prelu', name = name) + #ignore param act_type, set it in this function + if act_type == 'prelu': + body = mx.sym.LeakyReLU(data=data, act_type='prelu', name=name) else: - body = mx.sym.Activation(data=data, act_type=act_type, name=name) + body = mx.sym.Activation(data=data, act_type=act_type, name=name) return body + bn_mom = config.bn_mom -def Linear(data, num_filter=1, kernel=(1, 1), stride=(1, 1), pad=(0, 0), num_group=1, name=None, suffix=''): - conv = mx.sym.Convolution(data=data, num_filter=num_filter, kernel=kernel, num_group=num_group, stride=stride, pad=pad, no_bias=True, name='%s%s_conv2d' %(name, suffix)) - bn = mx.sym.BatchNorm(data=conv, name='%s%s_batchnorm' %(name, suffix), fix_gamma=False,momentum=bn_mom) + + +def Linear(data, + num_filter=1, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + num_group=1, + name=None, + suffix=''): + conv = mx.sym.Convolution(data=data, + num_filter=num_filter, + kernel=kernel, + num_group=num_group, + stride=stride, + pad=pad, + no_bias=True, + name='%s%s_conv2d' % (name, suffix)) + bn = mx.sym.BatchNorm(data=conv, + name='%s%s_batchnorm' % (name, suffix), + fix_gamma=False, + momentum=bn_mom) return bn + def get_fc1(last_conv, num_classes, fc_type, input_channel=512): - body = last_conv - if fc_type=='Z': - body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn1') - body = mx.symbol.Dropout(data=body, p=0.4) - fc1 = body - elif fc_type=='E': - body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn1') - body = mx.symbol.Dropout(data=body, p=0.4) - fc1 = mx.sym.FullyConnected(data=body, num_hidden=num_classes, name='pre_fc1') - fc1 = mx.sym.BatchNorm(data=fc1, fix_gamma=True, eps=2e-5, momentum=bn_mom, name='fc1') - elif fc_type=='FC': - body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn1') - fc1 = mx.sym.FullyConnected(data=body, num_hidden=num_classes, name='pre_fc1') - fc1 = mx.sym.BatchNorm(data=fc1, fix_gamma=True, eps=2e-5, momentum=bn_mom, name='fc1') - elif fc_type=='SFC': - body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn1') - body = Conv(data=body, num_filter=input_channel, kernel=(3,3), stride=(2,2), pad=(1,1), - no_bias=True, name="convf", num_group = input_channel) - body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bnf') - body = Act(data=body, act_type=config.net_act, name='reluf') - body = Conv(data=body, num_filter=input_channel, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="convf2") - body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bnf2') - body = Act(data=body, act_type=config.net_act, name='reluf2') - fc1 = mx.sym.FullyConnected(data=body, num_hidden=num_classes, name='pre_fc1') - fc1 = mx.sym.BatchNorm(data=fc1, fix_gamma=True, eps=2e-5, momentum=bn_mom, name='fc1') - elif fc_type=='GAP': - bn1 = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn1') - relu1 = Act(data=bn1, act_type=config.net_act, name='relu1') - # Although kernel is not used here when global_pool=True, we should put one - pool1 = mx.sym.Pooling(data=relu1, global_pool=True, kernel=(7, 7), pool_type='avg', name='pool1') - flat = mx.sym.Flatten(data=pool1) - fc1 = mx.sym.FullyConnected(data=flat, num_hidden=num_classes, name='pre_fc1') - fc1 = mx.sym.BatchNorm(data=fc1, fix_gamma=True, eps=2e-5, momentum=bn_mom, name='fc1') - elif fc_type=='GNAP': #mobilefacenet++ - filters_in = 512 # param in mobilefacenet - if num_classes>filters_in: - body = mx.sym.Convolution(data=last_conv, num_filter=num_classes, kernel=(1,1), stride=(1,1), pad=(0,0), no_bias=True, name='convx') - body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=0.9, name='convx_bn') - body = Act(data=body, act_type=config.net_act, name='convx_relu') - filters_in = num_classes - else: - body = last_conv - body = mx.sym.BatchNorm(data=body, fix_gamma=True, eps=2e-5, momentum=0.9, name='bn6f') + body = last_conv + if fc_type == 'Z': + body = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name='bn1') + body = mx.symbol.Dropout(data=body, p=0.4) + fc1 = body + elif fc_type == 'E': + body = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name='bn1') + body = mx.symbol.Dropout(data=body, p=0.4) + fc1 = mx.sym.FullyConnected(data=body, + num_hidden=num_classes, + name='pre_fc1') + fc1 = mx.sym.BatchNorm(data=fc1, + fix_gamma=True, + eps=2e-5, + momentum=bn_mom, + name='fc1') + elif fc_type == 'FC': + body = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name='bn1') + fc1 = mx.sym.FullyConnected(data=body, + num_hidden=num_classes, + name='pre_fc1') + fc1 = mx.sym.BatchNorm(data=fc1, + fix_gamma=True, + eps=2e-5, + momentum=bn_mom, + name='fc1') + elif fc_type == 'SFC': + body = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name='bn1') + body = Conv(data=body, + num_filter=input_channel, + kernel=(3, 3), + stride=(2, 2), + pad=(1, 1), + no_bias=True, + name="convf", + num_group=input_channel) + body = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name='bnf') + body = Act(data=body, act_type=config.net_act, name='reluf') + body = Conv(data=body, + num_filter=input_channel, + kernel=(1, 1), + pad=(0, 0), + stride=(1, 1), + name="convf2") + body = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name='bnf2') + body = Act(data=body, act_type=config.net_act, name='reluf2') + fc1 = mx.sym.FullyConnected(data=body, + num_hidden=num_classes, + name='pre_fc1') + fc1 = mx.sym.BatchNorm(data=fc1, + fix_gamma=True, + eps=2e-5, + momentum=bn_mom, + name='fc1') + elif fc_type == 'GAP': + bn1 = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name='bn1') + relu1 = Act(data=bn1, act_type=config.net_act, name='relu1') + # Although kernel is not used here when global_pool=True, we should put one + pool1 = mx.sym.Pooling(data=relu1, + global_pool=True, + kernel=(7, 7), + pool_type='avg', + name='pool1') + flat = mx.sym.Flatten(data=pool1) + fc1 = mx.sym.FullyConnected(data=flat, + num_hidden=num_classes, + name='pre_fc1') + fc1 = mx.sym.BatchNorm(data=fc1, + fix_gamma=True, + eps=2e-5, + momentum=bn_mom, + name='fc1') + elif fc_type == 'GNAP': #mobilefacenet++ + filters_in = 512 # param in mobilefacenet + if num_classes > filters_in: + body = mx.sym.Convolution(data=last_conv, + num_filter=num_classes, + kernel=(1, 1), + stride=(1, 1), + pad=(0, 0), + no_bias=True, + name='convx') + body = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=0.9, + name='convx_bn') + body = Act(data=body, act_type=config.net_act, name='convx_relu') + filters_in = num_classes + else: + body = last_conv + body = mx.sym.BatchNorm(data=body, + fix_gamma=True, + eps=2e-5, + momentum=0.9, + name='bn6f') - spatial_norm=body*body - spatial_norm=mx.sym.sum(data=spatial_norm, axis=1, keepdims=True) - spatial_sqrt=mx.sym.sqrt(spatial_norm) - #spatial_mean=mx.sym.mean(spatial_sqrt, axis=(1,2,3), keepdims=True) - spatial_mean=mx.sym.mean(spatial_sqrt) - spatial_div_inverse=mx.sym.broadcast_div(spatial_mean, spatial_sqrt) + spatial_norm = body * body + spatial_norm = mx.sym.sum(data=spatial_norm, axis=1, keepdims=True) + spatial_sqrt = mx.sym.sqrt(spatial_norm) + #spatial_mean=mx.sym.mean(spatial_sqrt, axis=(1,2,3), keepdims=True) + spatial_mean = mx.sym.mean(spatial_sqrt) + spatial_div_inverse = mx.sym.broadcast_div(spatial_mean, spatial_sqrt) - spatial_attention_inverse=mx.symbol.tile(spatial_div_inverse, reps=(1,filters_in,1,1)) - body=body*spatial_attention_inverse - #body = mx.sym.broadcast_mul(body, spatial_div_inverse) + spatial_attention_inverse = mx.symbol.tile(spatial_div_inverse, + reps=(1, filters_in, 1, 1)) + body = body * spatial_attention_inverse + #body = mx.sym.broadcast_mul(body, spatial_div_inverse) + + fc1 = mx.sym.Pooling(body, + kernel=(7, 7), + global_pool=True, + pool_type='avg') + if num_classes < filters_in: + fc1 = mx.sym.BatchNorm(data=fc1, + fix_gamma=True, + eps=2e-5, + momentum=0.9, + name='bn6w') + fc1 = mx.sym.FullyConnected(data=fc1, + num_hidden=num_classes, + name='pre_fc1') + else: + fc1 = mx.sym.Flatten(data=fc1) + fc1 = mx.sym.BatchNorm(data=fc1, + fix_gamma=True, + eps=2e-5, + momentum=0.9, + name='fc1') + elif fc_type == "GDC": #mobilefacenet_v1 + conv_6_dw = Linear(last_conv, + num_filter=input_channel, + num_group=input_channel, + kernel=(7, 7), + pad=(0, 0), + stride=(1, 1), + name="conv_6dw7_7") + conv_6_f = mx.sym.FullyConnected(data=conv_6_dw, + num_hidden=num_classes, + name='pre_fc1') + fc1 = mx.sym.BatchNorm(data=conv_6_f, + fix_gamma=True, + eps=2e-5, + momentum=bn_mom, + name='fc1') + elif fc_type == 'F': + body = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name='bn1') + body = mx.symbol.Dropout(data=body, p=0.4) + fc1 = mx.sym.FullyConnected(data=body, + num_hidden=num_classes, + name='fc1') + elif fc_type == 'G': + body = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name='bn1') + fc1 = mx.sym.FullyConnected(data=body, + num_hidden=num_classes, + name='fc1') + elif fc_type == 'H': + fc1 = mx.sym.FullyConnected(data=body, + num_hidden=num_classes, + name='fc1') + elif fc_type == 'I': + body = mx.sym.BatchNorm(data=body, + fix_gamma=False, + eps=2e-5, + momentum=bn_mom, + name='bn1') + fc1 = mx.sym.FullyConnected(data=body, + num_hidden=num_classes, + name='pre_fc1') + fc1 = mx.sym.BatchNorm(data=fc1, + fix_gamma=True, + eps=2e-5, + momentum=bn_mom, + name='fc1') + elif fc_type == 'J': + fc1 = mx.sym.FullyConnected(data=body, + num_hidden=num_classes, + name='pre_fc1') + fc1 = mx.sym.BatchNorm(data=fc1, + fix_gamma=True, + eps=2e-5, + momentum=bn_mom, + name='fc1') + return fc1 - fc1 = mx.sym.Pooling(body, kernel=(7, 7), global_pool=True, pool_type='avg') - if num_classes= len(self.seq): - raise StopIteration - idx = self.seq[self.cur] - self.cur += 1 - if self.imgrec is not None: - s = self.imgrec.read_idx(idx) - header, img = recordio.unpack(s) - label = header.label - return label, img, None, None - else: - label, fname, bbox, landmark = self.imglist[idx] - return label, self.read_image(fname), bbox, landmark - else: - s = self.imgrec.read() - if s is None: - raise StopIteration - header, img = recordio.unpack(s) - return header.label, img, None, None - - def brightness_aug(self, src, x): - alpha = 1.0 + random.uniform(-x, x) - src *= alpha - return src - - def contrast_aug(self, src, x): - alpha = 1.0 + random.uniform(-x, x) - coef = np.array([[[0.299, 0.587, 0.114]]]) - gray = src * coef - gray = (3.0 * (1.0 - alpha) / gray.size) * np.sum(gray) - src *= alpha - src += gray - return src - - def saturation_aug(self, src, x): - alpha = 1.0 + random.uniform(-x, x) - coef = np.array([[[0.299, 0.587, 0.114]]]) - gray = src * coef - gray = np.sum(gray, axis=2, keepdims=True) - gray *= (1.0 - alpha) - src *= alpha - src += gray - return src - - def color_aug(self, img, x): - augs = [self.brightness_aug, self.contrast_aug, self.saturation_aug] - random.shuffle(augs) - for aug in augs: - #print(img.shape) - img = aug(img, x) - #print(img.shape) - return img - - def mirror_aug(self, img): - _rd = random.randint(0,1) - if _rd==1: - for c in xrange(img.shape[2]): - img[:,:,c] = np.fliplr(img[:,:,c]) - return img - - - def next(self): - if not self.is_init: - self.reset() - self.is_init = True - """Returns the next batch of data.""" - #print('in next', self.cur, self.labelcur) - self.nbatch+=1 - batch_size = self.batch_size - c, h, w = self.data_shape - batch_data = nd.empty((batch_size, c, h, w)) - if self.provide_label is not None: - batch_label = nd.empty(self.provide_label[0][1]) - i = 0 - try: - while i < batch_size: - label, s, bbox, landmark = self.next_sample() - #if label[1]>=0.0 or label[2]>=0.0: - # print(label[0:10]) - _data = self.imdecode(s) - if self.rand_mirror: - _rd = random.randint(0,1) - if _rd==1: - _data = mx.ndarray.flip(data=_data, axis=1) - if self.nd_mean is not None: - _data = _data.astype('float32') - _data -= self.nd_mean - _data *= 0.0078125 - if self.cutoff>0: - centerh = random.randint(0, _data.shape[0]-1) - centerw = random.randint(0, _data.shape[1]-1) - half = self.cutoff//2 - starth = max(0, centerh-half) - endh = min(_data.shape[0], centerh+half) - startw = max(0, centerw-half) - endw = min(_data.shape[1], centerw+half) - _data = _data.astype('float32') - #print(starth, endh, startw, endw, _data.shape) - _data[starth:endh, startw:endw, :] = 127.5 - data = [_data] - try: - self.check_valid_image(data) - except RuntimeError as e: - logging.debug('Invalid image, skipping: %s', str(e)) - continue - #print('aa',data[0].shape) - #data = self.augmentation_transform(data) - #print('bb',data[0].shape) - for datum in data: - assert i < batch_size, 'Batch size must be multiples of augmenter output length' - #print(datum.shape) - batch_data[i][:] = self.postprocess_data(datum) - batch_label[i][:] = label - i += 1 - except StopIteration: - if i>> dataIter.read_image('Face.jpg') # returns decoded raw bytes. - """ - with open(os.path.join(self.path_root, fname), 'rb') as fin: - img = fin.read() - return img - - def augmentation_transform(self, data): - """Transforms input data with specified augmentation.""" - for aug in self.auglist: - data = [ret for src in data for ret in aug(src)] - return data - - def postprocess_data(self, datum): - """Final postprocessing step before image is loaded into the batch.""" - return nd.transpose(datum, axes=(2, 0, 1)) - -class FaceImageIterList(io.DataIter): - def __init__(self, iter_list): - assert len(iter_list)>0 - self.provide_data = iter_list[0].provide_data - self.provide_label = iter_list[0].provide_label - self.iter_list = iter_list - self.cur_iter = None - - def reset(self): - self.cur_iter.reset() - - def next(self): - self.cur_iter = random.choice(self.iter_list) - while True: - try: - ret = self.cur_iter.next() - except StopIteration: - self.cur_iter.reset() - continue - return ret - - diff --git a/src/align/__init__.py b/src/align/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/align/align_celeb.py b/src/align/align_celeb.py deleted file mode 100644 index ae83b06..0000000 --- a/src/align/align_celeb.py +++ /dev/null @@ -1,225 +0,0 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from scipy import misc -import sys -import os -import cv2 -import argparse -import tensorflow as tf -import numpy as np -import base64 -#import facenet -import detect_face -from easydict import EasyDict as edict -import random -from time import sleep -sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'common')) -import face_image -import face_preprocess - -def to_rgb(img): - w, h = img.shape - ret = np.empty((w, h, 3), dtype=np.uint8) - ret[:, :, 0] = ret[:, :, 1] = ret[:, :, 2] = img - return ret - - -def IOU(Reframe,GTframe): - x1 = Reframe[0]; - y1 = Reframe[1]; - width1 = Reframe[2]-Reframe[0]; - height1 = Reframe[3]-Reframe[1]; - - x2 = GTframe[0] - y2 = GTframe[1] - width2 = GTframe[2]-GTframe[0] - height2 = GTframe[3]-GTframe[1] - - endx = max(x1+width1,x2+width2) - startx = min(x1,x2) - width = width1+width2-(endx-startx) - - endy = max(y1+height1,y2+height2) - starty = min(y1,y2) - height = height1+height2-(endy-starty) - - if width <=0 or height <= 0: - ratio = 0 - else: - Area = width*height - Area1 = width1*height1 - Area2 = width2*height2 - ratio = Area*1./(Area1+Area2-Area) - return ratio - - -def main(args): - output_dir = os.path.expanduser(args.output_dir) - if not os.path.exists(output_dir): - os.makedirs(output_dir) - datamap = {} - pp = 0 - datasize = 0 - verr = 0 - for line in open(args.input_dir+"_clean_list.txt", 'r'): - pp+=1 - if pp%10000==0: - print('loading list', pp) - line = line.strip()[2:] - if not line.startswith('m.'): - continue - vec = line.split('/') - assert len(vec)==2 - #print(line) - person = vec[0] - img = vec[1] - try: - img_id = int(img.split('.')[0]) - except ValueError: - #print('value error', line) - verr+=1 - continue - if not person in datamap: - labelid = len(datamap) - datamap[person] = [labelid, {img_id : 1}] - else: - datamap[person][1][img_id] = 1 - datasize+=1 - - print('dataset size', args.name, datasize) - print('dataset err', verr) - - print('Creating networks and loading parameters') - - with tf.Graph().as_default(): - #gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=args.gpu_memory_fraction) - #sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options, log_device_placement=False)) - sess = tf.Session() - with sess.as_default(): - pnet, rnet, onet = detect_face.create_mtcnn(sess, None) - - minsize = 100 # minimum size of face - #threshold = [ 0.6, 0.7, 0.7 ] # three steps's threshold - threshold = [ 0.6, 0.6, 0.3 ] # three steps's threshold - factor = 0.709 # scale factor - - print(minsize) - print(threshold) - print(factor) - - # Add a random key to the filename to allow alignment using multiple processes - #random_key = np.random.randint(0, high=99999) - #bounding_boxes_filename = os.path.join(output_dir, 'bounding_boxes_%05d.txt' % random_key) - output_filename = os.path.join(output_dir, 'faceinsight_align_%s.lst' % args.name) - - with open(output_filename, "w") as text_file: - nrof_images_total = 0 - nrof_successfully_aligned = 0 - nrof_changed = 0 - nrof_iou3 = 0 - nrof_force = 0 - for line in open(args.input_dir, 'r'): - vec = line.strip().split() - person = vec[0] - img_id = int(vec[1]) - v = datamap.get(person, None) - if v is None: - continue - #TODO - #if not img_id in v[1]: - # continue - labelid = v[0] - img_str = base64.b64decode(vec[-1]) - nparr = np.fromstring(img_str, np.uint8) - img = cv2.imdecode(nparr, cv2.CV_LOAD_IMAGE_COLOR) - img = img[...,::-1] #to rgb - if nrof_images_total%100==0: - print("Processing %d, (%d)" % (nrof_images_total, nrof_successfully_aligned)) - nrof_images_total += 1 - target_dir = os.path.join(output_dir, person) - if not os.path.exists(target_dir): - os.makedirs(target_dir) - target_path = os.path.join(target_dir, "%d.jpg"%img_id) - _minsize = minsize - fimage = edict() - fimage.bbox = None - fimage.image_path = target_path - fimage.classname = str(labelid) - if fimage.bbox is not None: - _bb = fimage.bbox - _minsize = min( [_bb[2]-_bb[0], _bb[3]-_bb[1], img.shape[0]//2, img.shape[1]//2] ) - else: - _minsize = min(img.shape[0]//5, img.shape[1]//5) - bounding_boxes, points = detect_face.detect_face(img, _minsize, pnet, rnet, onet, threshold, factor) - bindex = -1 - nrof_faces = bounding_boxes.shape[0] - if fimage.bbox is None and nrof_faces>0: - det = bounding_boxes[:,0:4] - img_size = np.asarray(img.shape)[0:2] - bindex = 0 - if nrof_faces>1: - bounding_box_size = (det[:,2]-det[:,0])*(det[:,3]-det[:,1]) - img_center = img_size / 2 - offsets = np.vstack([ (det[:,0]+det[:,2])/2-img_center[1], (det[:,1]+det[:,3])/2-img_center[0] ]) - offset_dist_squared = np.sum(np.power(offsets,2.0),0) - bindex = np.argmax(bounding_box_size-offset_dist_squared*2.0) # some extra weight on the centering - if fimage.bbox is not None: - if nrof_faces>0: - assert(bounding_boxes.shape[0]==points.shape[1]) - det = bounding_boxes[:,0:4] - img_size = np.asarray(img.shape)[0:2] - index2 = [0.0, 0] - for i in xrange(det.shape[0]): - _det = det[i] - iou = IOU(fimage.bbox, _det) - if iou>index2[0]: - index2[0] = iou - index2[1] = i - if index2[0]>-0.3: - bindex = index2[1] - nrof_iou3+=1 - if bindex<0: - bounding_boxes, points = detect_face.detect_face_force(img, fimage.bbox, pnet, rnet, onet) - bindex = 0 - nrof_force+=1 - - if bindex>=0: - - det = bounding_boxes[:,0:4] - det = det[bindex,:] - points = points[:, bindex] - landmark = points.reshape((2,5)).T - #points need to be transpose, points = points.reshape( (5,2) ).transpose() - det = np.squeeze(det) - bb = det - points = list(points.flatten()) - assert(len(points)==10) - warped = face_preprocess.preprocess(img, bbox=bb, landmark = landmark, image_size=args.image_size) - misc.imsave(target_path, warped) - nrof_successfully_aligned += 1 - oline = '%d\t%s\t%d' % (1,fimage.image_path, int(fimage.classname)) - #oline = '%d\t%s\t%d\t%d\t%d\t%d\t%d\t' % (0,fimage.image_path, int(fimage.classname), bb[0], bb[1], bb[2], bb[3]) - #oline += '\t'.join([str(x) for x in points]) - text_file.write("%s\n"%oline) - - print('Total number of images: %d' % nrof_images_total) - print('Number of successfully aligned images: %d' % nrof_successfully_aligned) - print('Number of changed: %d' % nrof_changed) - print('Number of iou3: %d' % nrof_iou3) - print('Number of force: %d' % nrof_force) - -def parse_arguments(argv): - parser = argparse.ArgumentParser() - - parser.add_argument('--input-dir', type=str, help='Directory with unaligned images.') - parser.add_argument('--name', type=str, default='celeb', help='') - parser.add_argument('--output-dir', type=str, help='Directory with aligned face thumbnails.') - parser.add_argument('--image-size', type=str, help='Image size (height, width) in pixels.', default='112,112') - #parser.add_argument('--margin', type=int, - # help='Margin for the crop around the bounding box (height, width) in pixels.', default=44) - return parser.parse_args(argv) - -if __name__ == '__main__': - main(parse_arguments(sys.argv[1:])) diff --git a/src/align/align_dataset.py b/src/align/align_dataset.py deleted file mode 100644 index e74224a..0000000 --- a/src/align/align_dataset.py +++ /dev/null @@ -1,137 +0,0 @@ -"""Performs face alignment and stores face thumbnails in the output directory.""" - -# MIT License -# -# Copyright (c) 2016 David Sandberg -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from scipy import misc -import sys -import os -import argparse -import random -import align_dlib # @UnresolvedImport -import facenet - -def main(args): - align = align_dlib.AlignDlib(os.path.expanduser(args.dlib_face_predictor)) - landmarkIndices = align_dlib.AlignDlib.OUTER_EYES_AND_NOSE - output_dir = os.path.expanduser(args.output_dir) - if not os.path.exists(output_dir): - os.makedirs(output_dir) - # Store some git revision info in a text file in the log directory - src_path,_ = os.path.split(os.path.realpath(__file__)) - facenet.store_revision_info(src_path, output_dir, ' '.join(sys.argv)) - dataset = facenet.get_dataset(args.input_dir) - random.shuffle(dataset) - # Scale the image such that the face fills the frame when cropped to crop_size - scale = float(args.face_size) / args.image_size - nrof_images_total = 0 - nrof_prealigned_images = 0 - nrof_successfully_aligned = 0 - for cls in dataset: - output_class_dir = os.path.join(output_dir, cls.name) - if not os.path.exists(output_class_dir): - os.makedirs(output_class_dir) - random.shuffle(cls.image_paths) - for image_path in cls.image_paths: - nrof_images_total += 1 - filename = os.path.splitext(os.path.split(image_path)[1])[0] - output_filename = os.path.join(output_class_dir, filename+'.png') - if not os.path.exists(output_filename): - try: - img = misc.imread(image_path) - except (IOError, ValueError, IndexError) as e: - errorMessage = '{}: {}'.format(image_path, e) - print(errorMessage) - else: - if img.ndim == 2: - img = facenet.to_rgb(img) - if args.use_center_crop: - scaled = misc.imresize(img, args.prealigned_scale, interp='bilinear') - sz1 = scaled.shape[1]/2 - sz2 = args.image_size/2 - aligned = scaled[(sz1-sz2):(sz1+sz2),(sz1-sz2):(sz1+sz2),:] - else: - aligned = align.align(args.image_size, img, landmarkIndices=landmarkIndices, - skipMulti=False, scale=scale) - if aligned is not None: - print(image_path) - nrof_successfully_aligned += 1 - misc.imsave(output_filename, aligned) - elif args.prealigned_dir: - # Face detection failed. Use center crop from pre-aligned dataset - class_name = os.path.split(output_class_dir)[1] - image_path_without_ext = os.path.join(os.path.expanduser(args.prealigned_dir), - class_name, filename) - # Find the extension of the image - exts = ('jpg', 'png') - for ext in exts: - temp_path = image_path_without_ext + '.' + ext - image_path = '' - if os.path.exists(temp_path): - image_path = temp_path - break - try: - img = misc.imread(image_path) - except (IOError, ValueError, IndexError) as e: - errorMessage = '{}: {}'.format(image_path, e) - print(errorMessage) - else: - scaled = misc.imresize(img, args.prealigned_scale, interp='bilinear') - sz1 = scaled.shape[1]/2 - sz2 = args.image_size/2 - cropped = scaled[(sz1-sz2):(sz1+sz2),(sz1-sz2):(sz1+sz2),:] - print(image_path) - nrof_prealigned_images += 1 - misc.imsave(output_filename, cropped) - else: - print('Unable to align "%s"' % image_path) - - print('Total number of images: %d' % nrof_images_total) - print('Number of successfully aligned images: %d' % nrof_successfully_aligned) - print('Number of pre-aligned images: %d' % nrof_prealigned_images) - - -def parse_arguments(argv): - parser = argparse.ArgumentParser() - - parser.add_argument('input_dir', type=str, help='Directory with unaligned images.') - parser.add_argument('output_dir', type=str, help='Directory with aligned face thumbnails.') - parser.add_argument('--dlib_face_predictor', type=str, - help='File containing the dlib face predictor.', default='../data/shape_predictor_68_face_landmarks.dat') - parser.add_argument('--image_size', type=int, - help='Image size (height, width) in pixels.', default=110) - parser.add_argument('--face_size', type=int, - help='Size of the face thumbnail (height, width) in pixels.', default=96) - parser.add_argument('--use_center_crop', - help='Use the center crop of the original image after scaling the image using prealigned_scale.', action='store_true') - parser.add_argument('--prealigned_dir', type=str, - help='Replace image with a pre-aligned version when face detection fails.', default='') - parser.add_argument('--prealigned_scale', type=float, - help='The amount of scaling to apply to prealigned images before taking the center crop.', default=0.87) - return parser.parse_args(argv) - -if __name__ == '__main__': - main(parse_arguments(sys.argv[1:])) diff --git a/src/align/align_dataset_mtcnn.py b/src/align/align_dataset_mtcnn.py deleted file mode 100644 index d2a3eea..0000000 --- a/src/align/align_dataset_mtcnn.py +++ /dev/null @@ -1,143 +0,0 @@ -"""Performs face alignment and stores face thumbnails in the output directory.""" -# MIT License -# -# Copyright (c) 2016 David Sandberg -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from scipy import misc -import sys -import os -import argparse -import tensorflow as tf -import numpy as np -import facenet -import align.detect_face -import random -from time import sleep - -def main(args): - sleep(random.random()) - output_dir = os.path.expanduser(args.output_dir) - if not os.path.exists(output_dir): - os.makedirs(output_dir) - # Store some git revision info in a text file in the log directory - src_path,_ = os.path.split(os.path.realpath(__file__)) - facenet.store_revision_info(src_path, output_dir, ' '.join(sys.argv)) - dataset = facenet.get_dataset(args.input_dir) - - print('Creating networks and loading parameters') - - with tf.Graph().as_default(): - gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=args.gpu_memory_fraction) - sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options, log_device_placement=False)) - with sess.as_default(): - pnet, rnet, onet = align.detect_face.create_mtcnn(sess, None) - - minsize = 20 # minimum size of face - threshold = [ 0.6, 0.7, 0.7 ] # three steps's threshold - factor = 0.709 # scale factor - - # Add a random key to the filename to allow alignment using multiple processes - random_key = np.random.randint(0, high=99999) - bounding_boxes_filename = os.path.join(output_dir, 'bounding_boxes_%05d.txt' % random_key) - - with open(bounding_boxes_filename, "w") as text_file: - nrof_images_total = 0 - nrof_successfully_aligned = 0 - if args.random_order: - random.shuffle(dataset) - for cls in dataset: - output_class_dir = os.path.join(output_dir, cls.name) - if not os.path.exists(output_class_dir): - os.makedirs(output_class_dir) - if args.random_order: - random.shuffle(cls.image_paths) - for image_path in cls.image_paths: - nrof_images_total += 1 - filename = os.path.splitext(os.path.split(image_path)[1])[0] - output_filename = os.path.join(output_class_dir, filename+'.png') - print(image_path) - if not os.path.exists(output_filename): - try: - img = misc.imread(image_path) - except (IOError, ValueError, IndexError) as e: - errorMessage = '{}: {}'.format(image_path, e) - print(errorMessage) - else: - if img.ndim<2: - print('Unable to align "%s"' % image_path) - text_file.write('%s\n' % (output_filename)) - continue - if img.ndim == 2: - img = facenet.to_rgb(img) - img = img[:,:,0:3] - - bounding_boxes, _ = align.detect_face.detect_face(img, minsize, pnet, rnet, onet, threshold, factor) - nrof_faces = bounding_boxes.shape[0] - if nrof_faces>0: - det = bounding_boxes[:,0:4] - img_size = np.asarray(img.shape)[0:2] - if nrof_faces>1: - bounding_box_size = (det[:,2]-det[:,0])*(det[:,3]-det[:,1]) - img_center = img_size / 2 - offsets = np.vstack([ (det[:,0]+det[:,2])/2-img_center[1], (det[:,1]+det[:,3])/2-img_center[0] ]) - offset_dist_squared = np.sum(np.power(offsets,2.0),0) - index = np.argmax(bounding_box_size-offset_dist_squared*2.0) # some extra weight on the centering - det = det[index,:] - det = np.squeeze(det) - bb = np.zeros(4, dtype=np.int32) - bb[0] = np.maximum(det[0]-args.margin/2, 0) - bb[1] = np.maximum(det[1]-args.margin/2, 0) - bb[2] = np.minimum(det[2]+args.margin/2, img_size[1]) - bb[3] = np.minimum(det[3]+args.margin/2, img_size[0]) - cropped = img[bb[1]:bb[3],bb[0]:bb[2],:] - scaled = misc.imresize(cropped, (args.image_size, args.image_size), interp='bilinear') - nrof_successfully_aligned += 1 - misc.imsave(output_filename, scaled) - text_file.write('%s %d %d %d %d\n' % (output_filename, bb[0], bb[1], bb[2], bb[3])) - else: - print('Unable to align "%s"' % image_path) - text_file.write('%s\n' % (output_filename)) - - print('Total number of images: %d' % nrof_images_total) - print('Number of successfully aligned images: %d' % nrof_successfully_aligned) - - -def parse_arguments(argv): - parser = argparse.ArgumentParser() - - parser.add_argument('input_dir', type=str, help='Directory with unaligned images.') - parser.add_argument('output_dir', type=str, help='Directory with aligned face thumbnails.') - parser.add_argument('--image_size', type=int, - help='Image size (height, width) in pixels.', default=182) - parser.add_argument('--margin', type=int, - help='Margin for the crop around the bounding box (height, width) in pixels.', default=44) - parser.add_argument('--random_order', - help='Shuffles the order of images to enable alignment using multiple processes.', action='store_true') - parser.add_argument('--gpu_memory_fraction', type=float, - help='Upper bound on the amount of GPU memory that will be used by the process.', default=1.0) - return parser.parse_args(argv) - -if __name__ == '__main__': - main(parse_arguments(sys.argv[1:])) diff --git a/src/align/align_dlib.py b/src/align/align_dlib.py deleted file mode 100644 index e5e1337..0000000 --- a/src/align/align_dlib.py +++ /dev/null @@ -1,204 +0,0 @@ -# Copyright 2015-2016 Carnegie Mellon University -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Module for dlib-based alignment.""" - -# NOTE: This file has been copied from the openface project. -# https://github.com/cmusatyalab/openface/blob/master/openface/align_dlib.py - -import cv2 -import dlib -import numpy as np - -TEMPLATE = np.float32([ - (0.0792396913815, 0.339223741112), (0.0829219487236, 0.456955367943), - (0.0967927109165, 0.575648016728), (0.122141515615, 0.691921601066), - (0.168687863544, 0.800341263616), (0.239789390707, 0.895732504778), - (0.325662452515, 0.977068762493), (0.422318282013, 1.04329000149), - (0.531777802068, 1.06080371126), (0.641296298053, 1.03981924107), - (0.738105872266, 0.972268833998), (0.824444363295, 0.889624082279), - (0.894792677532, 0.792494155836), (0.939395486253, 0.681546643421), - (0.96111933829, 0.562238253072), (0.970579841181, 0.441758925744), - (0.971193274221, 0.322118743967), (0.163846223133, 0.249151738053), - (0.21780354657, 0.204255863861), (0.291299351124, 0.192367318323), - (0.367460241458, 0.203582210627), (0.4392945113, 0.233135599851), - (0.586445962425, 0.228141644834), (0.660152671635, 0.195923841854), - (0.737466449096, 0.182360984545), (0.813236546239, 0.192828009114), - (0.8707571886, 0.235293377042), (0.51534533827, 0.31863546193), - (0.516221448289, 0.396200446263), (0.517118861835, 0.473797687758), - (0.51816430343, 0.553157797772), (0.433701156035, 0.604054457668), - (0.475501237769, 0.62076344024), (0.520712933176, 0.634268222208), - (0.565874114041, 0.618796581487), (0.607054002672, 0.60157671656), - (0.252418718401, 0.331052263829), (0.298663015648, 0.302646354002), - (0.355749724218, 0.303020650651), (0.403718978315, 0.33867711083), - (0.352507175597, 0.349987615384), (0.296791759886, 0.350478978225), - (0.631326076346, 0.334136672344), (0.679073381078, 0.29645404267), - (0.73597236153, 0.294721285802), (0.782865376271, 0.321305281656), - (0.740312274764, 0.341849376713), (0.68499850091, 0.343734332172), - (0.353167761422, 0.746189164237), (0.414587777921, 0.719053835073), - (0.477677654595, 0.706835892494), (0.522732900812, 0.717092275768), - (0.569832064287, 0.705414478982), (0.635195811927, 0.71565572516), - (0.69951672331, 0.739419187253), (0.639447159575, 0.805236879972), - (0.576410514055, 0.835436670169), (0.525398405766, 0.841706377792), - (0.47641545769, 0.837505914975), (0.41379548902, 0.810045601727), - (0.380084785646, 0.749979603086), (0.477955996282, 0.74513234612), - (0.523389793327, 0.748924302636), (0.571057789237, 0.74332894691), - (0.672409137852, 0.744177032192), (0.572539621444, 0.776609286626), - (0.5240106503, 0.783370783245), (0.477561227414, 0.778476346951)]) - -INV_TEMPLATE = np.float32([ - (-0.04099179660567834, -0.008425234314031194, 2.575498465013183), - (0.04062510634554352, -0.009678089746831375, -1.2534351452524177), - (0.0003666902601348179, 0.01810332406086298, -0.32206331976076663)]) - -TPL_MIN, TPL_MAX = np.min(TEMPLATE, axis=0), np.max(TEMPLATE, axis=0) -MINMAX_TEMPLATE = (TEMPLATE - TPL_MIN) / (TPL_MAX - TPL_MIN) - - -class AlignDlib: - """ - Use `dlib's landmark estimation `_ to align faces. - - The alignment preprocess faces for input into a neural network. - Faces are resized to the same size (such as 96x96) and transformed - to make landmarks (such as the eyes and nose) appear at the same - location on every image. - - Normalized landmarks: - - .. image:: ../images/dlib-landmark-mean.png - """ - - #: Landmark indices corresponding to the inner eyes and bottom lip. - INNER_EYES_AND_BOTTOM_LIP = [39, 42, 57] - - #: Landmark indices corresponding to the outer eyes and nose. - OUTER_EYES_AND_NOSE = [36, 45, 33] - - def __init__(self, facePredictor): - """ - Instantiate an 'AlignDlib' object. - - :param facePredictor: The path to dlib's - :type facePredictor: str - """ - assert facePredictor is not None - - #pylint: disable=no-member - self.detector = dlib.get_frontal_face_detector() - self.predictor = dlib.shape_predictor(facePredictor) - - def getAllFaceBoundingBoxes(self, rgbImg): - """ - Find all face bounding boxes in an image. - - :param rgbImg: RGB image to process. Shape: (height, width, 3) - :type rgbImg: numpy.ndarray - :return: All face bounding boxes in an image. - :rtype: dlib.rectangles - """ - assert rgbImg is not None - - try: - return self.detector(rgbImg, 1) - except Exception as e: #pylint: disable=broad-except - print("Warning: {}".format(e)) - # In rare cases, exceptions are thrown. - return [] - - def getLargestFaceBoundingBox(self, rgbImg, skipMulti=False): - """ - Find the largest face bounding box in an image. - - :param rgbImg: RGB image to process. Shape: (height, width, 3) - :type rgbImg: numpy.ndarray - :param skipMulti: Skip image if more than one face detected. - :type skipMulti: bool - :return: The largest face bounding box in an image, or None. - :rtype: dlib.rectangle - """ - assert rgbImg is not None - - faces = self.getAllFaceBoundingBoxes(rgbImg) - if (not skipMulti and len(faces) > 0) or len(faces) == 1: - return max(faces, key=lambda rect: rect.width() * rect.height()) - else: - return None - - def findLandmarks(self, rgbImg, bb): - """ - Find the landmarks of a face. - - :param rgbImg: RGB image to process. Shape: (height, width, 3) - :type rgbImg: numpy.ndarray - :param bb: Bounding box around the face to find landmarks for. - :type bb: dlib.rectangle - :return: Detected landmark locations. - :rtype: list of (x,y) tuples - """ - assert rgbImg is not None - assert bb is not None - - points = self.predictor(rgbImg, bb) - #return list(map(lambda p: (p.x, p.y), points.parts())) - return [(p.x, p.y) for p in points.parts()] - - #pylint: disable=dangerous-default-value - def align(self, imgDim, rgbImg, bb=None, - landmarks=None, landmarkIndices=INNER_EYES_AND_BOTTOM_LIP, - skipMulti=False, scale=1.0): - r"""align(imgDim, rgbImg, bb=None, landmarks=None, landmarkIndices=INNER_EYES_AND_BOTTOM_LIP) - - Transform and align a face in an image. - - :param imgDim: The edge length in pixels of the square the image is resized to. - :type imgDim: int - :param rgbImg: RGB image to process. Shape: (height, width, 3) - :type rgbImg: numpy.ndarray - :param bb: Bounding box around the face to align. \ - Defaults to the largest face. - :type bb: dlib.rectangle - :param landmarks: Detected landmark locations. \ - Landmarks found on `bb` if not provided. - :type landmarks: list of (x,y) tuples - :param landmarkIndices: The indices to transform to. - :type landmarkIndices: list of ints - :param skipMulti: Skip image if more than one face detected. - :type skipMulti: bool - :param scale: Scale image before cropping to the size given by imgDim. - :type scale: float - :return: The aligned RGB image. Shape: (imgDim, imgDim, 3) - :rtype: numpy.ndarray - """ - assert imgDim is not None - assert rgbImg is not None - assert landmarkIndices is not None - - if bb is None: - bb = self.getLargestFaceBoundingBox(rgbImg, skipMulti) - if bb is None: - return - - if landmarks is None: - landmarks = self.findLandmarks(rgbImg, bb) - - npLandmarks = np.float32(landmarks) - npLandmarkIndices = np.array(landmarkIndices) - - #pylint: disable=maybe-no-member - H = cv2.getAffineTransform(npLandmarks[npLandmarkIndices], - imgDim * MINMAX_TEMPLATE[npLandmarkIndices]*scale + imgDim*(1-scale)/2) - thumbnail = cv2.warpAffine(rgbImg, H, (imgDim, imgDim)) - - return thumbnail diff --git a/src/align/align_facescrub.py b/src/align/align_facescrub.py deleted file mode 100644 index 890d0b0..0000000 --- a/src/align/align_facescrub.py +++ /dev/null @@ -1,289 +0,0 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from scipy import misc -import sys -import os -import json -import argparse -import tensorflow as tf -import numpy as np -#import facenet -import detect_face -import random -from time import sleep -sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'common')) -import face_image -from skimage import transform as trans -import cv2 - -def to_rgb(img): - w, h = img.shape - ret = np.empty((w, h, 3), dtype=np.uint8) - ret[:, :, 0] = ret[:, :, 1] = ret[:, :, 2] = img - return ret - - -def IOU(Reframe,GTframe): - x1 = Reframe[0]; - y1 = Reframe[1]; - width1 = Reframe[2]-Reframe[0]; - height1 = Reframe[3]-Reframe[1]; - - x2 = GTframe[0] - y2 = GTframe[1] - width2 = GTframe[2]-GTframe[0] - height2 = GTframe[3]-GTframe[1] - - endx = max(x1+width1,x2+width2) - startx = min(x1,x2) - width = width1+width2-(endx-startx) - - endy = max(y1+height1,y2+height2) - starty = min(y1,y2) - height = height1+height2-(endy-starty) - - if width <=0 or height <= 0: - ratio = 0 - else: - Area = width*height - Area1 = width1*height1 - Area2 = width2*height2 - ratio = Area*1./(Area1+Area2-Area) - return ratio - - -def main(args): - output_dir = os.path.expanduser(args.output_dir) - if not os.path.exists(output_dir): - os.makedirs(output_dir) - # Store some git revision info in a text file in the log directory - #facenet.store_revision_info(src_path, output_dir, ' '.join(sys.argv)) - image_dir = os.path.join(args.input_dir, 'facescrub') - dataset = face_image.get_dataset('facescrub', image_dir) - print('dataset size', len(dataset)) - bbox = {} - for label_file in ['facescrub_actors.txt', 'facescrub_actresses.txt']: - label_file = os.path.join(args.input_dir, label_file) - pp = 0 - for line in open(label_file, 'r'): - pp+=1 - if pp==1: - continue - vec = line.split("\t") - key = (vec[0], int(vec[2])) - value = [int(x) for x in vec[4].split(',')] - bbox[key] = value - print('bbox size', len(bbox)) - - valid_key = {} - json_data = open(os.path.join(args.input_dir, 'facescrub_uncropped_features_list.json')).read() - json_data = json.loads(json_data)['path'] - for _data in json_data: - key = _data.split('/')[-1] - pos = key.rfind('.') - if pos<0: - print(_data) - else: - key = key[0:pos] - keys = key.split('_') - #print(key) - if len(keys)!=2: - print('err', key, _data) - continue - #assert len(keys)==2 - key = (keys[0], int(keys[1])) - valid_key[key] = 1 - #print(key) - print('valid keys', len(valid_key)) - - print('Creating networks and loading parameters') - - with tf.Graph().as_default(): - #gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=args.gpu_memory_fraction) - #sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options, log_device_placement=False)) - sess = tf.Session() - with sess.as_default(): - pnet, rnet, onet = detect_face.create_mtcnn(sess, None) - - minsize = 100 # minimum size of face - threshold = [ 0.6, 0.7, 0.7 ] # three steps's threshold - factor = 0.709 # scale factor - image_size = [112,96] - image_size = [112,112] - src = np.array([ - [30.2946, 51.6963], - [65.5318, 51.5014], - [48.0252, 71.7366], - [33.5493, 92.3655], - [62.7299, 92.2041] ], dtype=np.float32 ) - if image_size[1]==112: - src[:,0] += 8.0 - - # Add a random key to the filename to allow alignment using multiple processes - #random_key = np.random.randint(0, high=99999) - #bounding_boxes_filename = os.path.join(output_dir, 'bounding_boxes_%05d.txt' % random_key) - #output_filename = os.path.join(output_dir, 'faceinsight_align_%s.lst' % args.name) - if not os.path.exists(args.output_dir): - os.makedirs(args.output_dir) - - output_filename = os.path.join(args.output_dir, 'lst') - - - with open(output_filename, "w") as text_file: - nrof_images_total = 0 - nrof = np.zeros( (5,), dtype=np.int32) - for fimage in dataset: - if nrof_images_total%100==0: - print("Processing %d, (%s)" % (nrof_images_total, nrof)) - nrof_images_total += 1 - #if nrof_images_total<950000: - # continue - image_path = fimage.image_path - if not os.path.exists(image_path): - print('image not found (%s)'%image_path) - continue - #print(image_path) - filename = os.path.splitext(os.path.split(image_path)[1])[0] - _paths = fimage.image_path.split('/') - print(fimage.image_path) - a,b = _paths[-2], _paths[-1] - pb = b.rfind('.') - bname = b[0:pb] - pb = bname.rfind('_') - body = bname[(pb+1):] - img_id = int(body) - key = (a, img_id) - if not key in valid_key: - continue - #print(b, img_id) - assert key in bbox - fimage.bbox = bbox[key] - try: - img = misc.imread(image_path) - except (IOError, ValueError, IndexError) as e: - errorMessage = '{}: {}'.format(image_path, e) - print(errorMessage) - else: - if img.ndim<2: - print('Unable to align "%s", img dim error' % image_path) - #text_file.write('%s\n' % (output_filename)) - continue - if img.ndim == 2: - img = to_rgb(img) - img = img[:,:,0:3] - tb = bname.replace(' ','_')+".png" - ta = a.replace(' ','_') - target_dir = os.path.join(args.output_dir, ta) - if not os.path.exists(target_dir): - os.makedirs(target_dir) - - target_file = os.path.join(target_dir, tb) - warped = None - if fimage.landmark is not None: - dst = fimage.landmark.astype(np.float32) - - tform = trans.SimilarityTransform() - tform.estimate(dst, src[0:3,:]*1.5+image_size[0]*0.25) - M = tform.params[0:2,:] - warped0 = cv2.warpAffine(img,M,(image_size[1]*2,image_size[0]*2), borderValue = 0.0) - _minsize = image_size[0] - bounding_boxes, points = detect_face.detect_face(warped0, _minsize, pnet, rnet, onet, threshold, factor) - if bounding_boxes.shape[0]>0: - bindex = 0 - det = bounding_boxes[bindex,0:4] - #points need to be transpose, points = points.reshape( (5,2) ).transpose() - dst = points[:, bindex].reshape( (2,5) ).T - tform = trans.SimilarityTransform() - tform.estimate(dst, src) - M = tform.params[0:2,:] - warped = cv2.warpAffine(warped0,M,(image_size[1],image_size[0]), borderValue = 0.0) - nrof[0]+=1 - #assert fimage.bbox is not None - if warped is None and fimage.bbox is not None: - _minsize = img.shape[0]//4 - bounding_boxes, points = detect_face.detect_face(img, _minsize, pnet, rnet, onet, threshold, factor) - if bounding_boxes.shape[0]>0: - det = bounding_boxes[:,0:4] - bindex = -1 - index2 = [0.0, 0] - for i in xrange(det.shape[0]): - _det = det[i] - iou = IOU(fimage.bbox, _det) - if iou>index2[0]: - index2[0] = iou - index2[1] = i - if index2[0]>0.3: - bindex = index2[1] - if bindex>=0: - dst = points[:, bindex].reshape( (2,5) ).T - tform = trans.SimilarityTransform() - tform.estimate(dst, src) - M = tform.params[0:2,:] - warped = cv2.warpAffine(img,M,(image_size[1],image_size[0]), borderValue = 0.0) - nrof[1]+=1 - #print('1',target_file,index2[0]) - if warped is None and fimage.bbox is not None: - bb = fimage.bbox - #croped = img[bb[1]:bb[3],bb[0]:bb[2],:] - bounding_boxes, points = detect_face.detect_face_force(img, bb, pnet, rnet, onet) - assert bounding_boxes.shape[0]==1 - _box = bounding_boxes[0] - if _box[4]>=0.3: - dst = points[:, 0].reshape( (2,5) ).T - tform = trans.SimilarityTransform() - tform.estimate(dst, src) - M = tform.params[0:2,:] - warped = cv2.warpAffine(img,M,(image_size[1],image_size[0]), borderValue = 0.0) - nrof[2]+=1 - #print('2',target_file) - - if warped is None: - roi = np.zeros( (4,), dtype=np.int32) - roi[0] = int(img.shape[1]*0.06) - roi[1] = int(img.shape[0]*0.06) - roi[2] = img.shape[1]-roi[0] - roi[3] = img.shape[0]-roi[1] - if fimage.bbox is not None: - bb = fimage.bbox - h = bb[3]-bb[1] - w = bb[2]-bb[0] - x = bb[0] - y = bb[1] - #roi = np.copy(bb) - _w = int( (float(h)/image_size[0])*image_size[1] ) - x += (w-_w)//2 - #x = min( max(0,x), img.shape[1] ) - x = max(0,x) - xw = x+_w - xw = min(xw, img.shape[1]) - roi = np.array( (x, y, xw, y+h), dtype=np.int32) - nrof[3]+=1 - else: - nrof[4]+=1 - #print('3',bb,roi,img.shape) - #print('3',target_file) - warped = img[roi[1]:roi[3],roi[0]:roi[2],:] - #print(warped.shape) - warped = cv2.resize(warped, (image_size[1], image_size[0])) - bgr = warped[...,::-1] - cv2.imwrite(target_file, bgr) - oline = '%d\t%s\t%d\n' % (1,target_file, int(fimage.classname)) - text_file.write(oline) - - -def parse_arguments(argv): - parser = argparse.ArgumentParser() - - parser.add_argument('--input-dir', type=str, help='Directory with unaligned images.') - parser.add_argument('--output-dir', type=str, help='Directory with aligned face thumbnails.') - #parser.add_argument('--image_size', type=int, - # help='Image size (height, width) in pixels.', default=182) - #parser.add_argument('--margin', type=int, - # help='Margin for the crop around the bounding box (height, width) in pixels.', default=44) - return parser.parse_args(argv) - -if __name__ == '__main__': - main(parse_arguments(sys.argv[1:])) - diff --git a/src/align/align_insight.py b/src/align/align_insight.py deleted file mode 100644 index 363bf3f..0000000 --- a/src/align/align_insight.py +++ /dev/null @@ -1,251 +0,0 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from scipy import misc -import sys -import os -import argparse -import tensorflow as tf -import numpy as np -#import facenet -import detect_face -import random -from time import sleep -sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'common')) -import face_image - -def to_rgb(img): - w, h = img.shape - ret = np.empty((w, h, 3), dtype=np.uint8) - ret[:, :, 0] = ret[:, :, 1] = ret[:, :, 2] = img - return ret - - -def IOU(Reframe,GTframe): - x1 = Reframe[0]; - y1 = Reframe[1]; - width1 = Reframe[2]-Reframe[0]; - height1 = Reframe[3]-Reframe[1]; - - x2 = GTframe[0] - y2 = GTframe[1] - width2 = GTframe[2]-GTframe[0] - height2 = GTframe[3]-GTframe[1] - - endx = max(x1+width1,x2+width2) - startx = min(x1,x2) - width = width1+width2-(endx-startx) - - endy = max(y1+height1,y2+height2) - starty = min(y1,y2) - height = height1+height2-(endy-starty) - - if width <=0 or height <= 0: - ratio = 0 - else: - Area = width*height - Area1 = width1*height1 - Area2 = width2*height2 - ratio = Area*1./(Area1+Area2-Area) - return ratio - - -def main(args): - output_dir = os.path.expanduser(args.output_dir) - if not os.path.exists(output_dir): - os.makedirs(output_dir) - # Store some git revision info in a text file in the log directory - src_path,_ = os.path.split(os.path.realpath(__file__)) - #facenet.store_revision_info(src_path, output_dir, ' '.join(sys.argv)) - dataset = face_image.get_dataset(args.name, args.input_dir) - print('dataset size', args.name, len(dataset)) - - print('Creating networks and loading parameters') - - with tf.Graph().as_default(): - #gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=args.gpu_memory_fraction) - #sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options, log_device_placement=False)) - sess = tf.Session() - with sess.as_default(): - pnet, rnet, onet = detect_face.create_mtcnn(sess, None) - - minsize = 100 # minimum size of face - threshold = [ 0.6, 0.7, 0.7 ] # three steps's threshold - factor = 0.709 # scale factor - if args.name=='lfw' or args.name=='webface' or args.name=='vgg': - minsize = 20 - threshold = [0.6,0.7,0.9] - factor = 0.85 - if args.name=='ytf': - minsize = 20 - threshold = [0.6,0.7,0.4] - factor = 0.85 - - print(minsize) - print(threshold) - print(factor) - - # Add a random key to the filename to allow alignment using multiple processes - #random_key = np.random.randint(0, high=99999) - #bounding_boxes_filename = os.path.join(output_dir, 'bounding_boxes_%05d.txt' % random_key) - output_filename = os.path.join(output_dir, 'faceinsight_align_%s.lst' % args.name) - - with open(output_filename, "w") as text_file: - nrof_images_total = 0 - nrof_successfully_aligned = 0 - nrof_changed = 0 - nrof_iou3 = 0 - nrof_force = 0 - for fimage in dataset: - if nrof_images_total%100==0: - print("Processing %d, (%d)" % (nrof_images_total, nrof_successfully_aligned)) - nrof_images_total += 1 - image_path = fimage.image_path - if not os.path.exists(image_path): - print('image not found (%s)'%image_path) - continue - filename = os.path.splitext(os.path.split(image_path)[1])[0] - #print(image_path) - try: - img = misc.imread(image_path) - except (IOError, ValueError, IndexError) as e: - errorMessage = '{}: {}'.format(image_path, e) - print(errorMessage) - else: - if img.ndim<2: - print('Unable to align "%s", img dim error' % image_path) - #text_file.write('%s\n' % (output_filename)) - continue - if img.ndim == 2: - img = to_rgb(img) - img = img[:,:,0:3] - _minsize = minsize - if fimage.bbox is not None: - _bb = fimage.bbox - _minsize = min( [_bb[2]-_bb[0], _bb[3]-_bb[1], img.shape[0]//2, img.shape[1]//2] ) - - bounding_boxes, points = detect_face.detect_face(img, _minsize, pnet, rnet, onet, threshold, factor) - bindex = -1 - nrof_faces = bounding_boxes.shape[0] - if fimage.bbox is None and nrof_faces>0: - det = bounding_boxes[:,0:4] - img_size = np.asarray(img.shape)[0:2] - bindex = 0 - if nrof_faces>1: - bounding_box_size = (det[:,2]-det[:,0])*(det[:,3]-det[:,1]) - img_center = img_size / 2 - offsets = np.vstack([ (det[:,0]+det[:,2])/2-img_center[1], (det[:,1]+det[:,3])/2-img_center[0] ]) - offset_dist_squared = np.sum(np.power(offsets,2.0),0) - bindex = np.argmax(bounding_box_size-offset_dist_squared*2.0) # some extra weight on the centering - if fimage.bbox is not None: - if nrof_faces>0: - assert(bounding_boxes.shape[0]==points.shape[1]) - det = bounding_boxes[:,0:4] - img_size = np.asarray(img.shape)[0:2] - index2 = [0.0, 0] - for i in xrange(det.shape[0]): - _det = det[i] - iou = IOU(fimage.bbox, _det) - if iou>index2[0]: - index2[0] = iou - index2[1] = i - if index2[0]>-0.3: - bindex = index2[1] - nrof_iou3+=1 - if bindex<0: - bounding_boxes, points = detect_face.detect_face_force(img, fimage.bbox, pnet, rnet, onet) - bindex = 0 - nrof_force+=1 - #if bindex<0: - # _img = img[fimage.bbox[1]:fimage.bbox[3], fimage.bbox[0]:fimage.bbox[2],:] - # woffset = fimage.bbox[0] - # hoffset = fimage.bbox[1] - # _minsize = min( [_img.shape[0]//3, _img.shape[1]//3] ) - # bounding_boxes, points = detect_face.detect_face(_img, _minsize, pnet, rnet, onet, [0.6,0.7,0.01], factor) - # nrof_faces = bounding_boxes.shape[0] - # print(nrof_faces) - # if nrof_faces>0: - # #print(points.shape) - # #assert(nrof_faces>0) - # bounding_boxes[:,0]+=woffset - # bounding_boxes[:,2]+=woffset - # bounding_boxes[:,1]+=hoffset - # bounding_boxes[:,3]+=hoffset - # points[0:5,:] += woffset - # points[5:10,:] += hoffset - # bindex = 0 - # score = bounding_boxes[bindex,4] - # print(score) - # if score<=0.0: - # bindex = -1 - # else: - # nrof_force+=1 - #if bindex<0: - # _bb = fimage.bbox - # _minsize = min( [_bb[2]-_bb[0], _bb[3]-_bb[1], img.shape[0]//2, img.shape[1]//2] ) - # bounding_boxes, points = detect_face.detect_face(img, _minsize, pnet, rnet, onet, [0.6,0.7,0.1], factor) - # nrof_faces = bounding_boxes.shape[0] - # print(nrof_faces) - # if nrof_faces>0: - # bindex = 0 - #if fimage.bbox is not None and bounding_boxes.shape[0]==0: - # bounding_boxes, points = detect_face.detect_face(img, _minsize, pnet, rnet, onet, [0.6,0.7,0.3], factor) - - - #print(bounding_boxes.shape, points.shape) - #print(nrof_faces, points.shape) - - if bindex>=0: - - det = bounding_boxes[:,0:4] - det = det[bindex,:] - points = points[:, bindex] - #points need to be transpose, points = points.reshape( (5,2) ).transpose() - det = np.squeeze(det) - #bb = np.zeros(4, dtype=np.int32) - #bb[0] = np.maximum(det[0]-args.margin/2, 0) - #bb[1] = np.maximum(det[1]-args.margin/2, 0) - #bb[2] = np.minimum(det[2]+args.margin/2, img_size[1]) - #bb[3] = np.minimum(det[3]+args.margin/2, img_size[0]) - bb = det - #print(points.shape) - points = list(points.flatten()) - assert(len(points)==10) - #cropped = img[bb[1]:bb[3],bb[0]:bb[2],:] - #scaled = misc.imresize(cropped, (args.image_size, args.image_size), interp='bilinear') - #misc.imsave(output_filename, scaled) - nrof_successfully_aligned += 1 - oline = '%d\t%s\t%d\t%d\t%d\t%d\t%d\t' % (0,fimage.image_path, int(fimage.classname), bb[0], bb[1], bb[2], bb[3]) - oline += '\t'.join([str(x) for x in points]) - text_file.write("%s\n"%oline) - else: - print('Unable to align "%s", no face detected' % image_path) - if args.force>0: - if fimage.bbox is None: - oline = '%d\t%s\t%d\n' % (0,fimage.image_path, int(fimage.classname)) - else: - bb = fimage.bbox - oline = '%d\t%s\t%d\t%d\t%d\t%d\t%d\n' % (0,fimage.image_path, int(fimage.classname), bb[0], bb[1], bb[2], bb[3]) - text_file.write(oline) - #text_file.write('%s\n' % (output_filename)) - - print('Total number of images: %d' % nrof_images_total) - print('Number of successfully aligned images: %d' % nrof_successfully_aligned) - print('Number of changed: %d' % nrof_changed) - print('Number of iou3: %d' % nrof_iou3) - print('Number of force: %d' % nrof_force) - -def parse_arguments(argv): - parser = argparse.ArgumentParser() - - parser.add_argument('--input-dir', type=str, help='Directory with unaligned images.') - parser.add_argument('--name', type=str, help='dataset name, can be facescrub, megaface, webface, celeb.') - parser.add_argument('--output-dir', type=str, help='Directory with aligned face thumbnails.') - parser.add_argument('--force', type=int, help='force to output if no faces detected.', default=1) - #parser.add_argument('--margin', type=int, - # help='Margin for the crop around the bounding box (height, width) in pixels.', default=44) - return parser.parse_args(argv) - -if __name__ == '__main__': - main(parse_arguments(sys.argv[1:])) diff --git a/src/align/align_lfw.py b/src/align/align_lfw.py deleted file mode 100644 index b80d887..0000000 --- a/src/align/align_lfw.py +++ /dev/null @@ -1,161 +0,0 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from scipy import misc -import sys -import os -import argparse -import tensorflow as tf -import numpy as np -#import facenet -import detect_face -import random -from time import sleep -sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'common')) -import face_image -import face_preprocess -from skimage import transform as trans -import cv2 - -def to_rgb(img): - w, h = img.shape - ret = np.empty((w, h, 3), dtype=np.uint8) - ret[:, :, 0] = ret[:, :, 1] = ret[:, :, 2] = img - return ret - - -def IOU(Reframe,GTframe): - x1 = Reframe[0]; - y1 = Reframe[1]; - width1 = Reframe[2]-Reframe[0]; - height1 = Reframe[3]-Reframe[1]; - - x2 = GTframe[0] - y2 = GTframe[1] - width2 = GTframe[2]-GTframe[0] - height2 = GTframe[3]-GTframe[1] - - endx = max(x1+width1,x2+width2) - startx = min(x1,x2) - width = width1+width2-(endx-startx) - - endy = max(y1+height1,y2+height2) - starty = min(y1,y2) - height = height1+height2-(endy-starty) - - if width <=0 or height <= 0: - ratio = 0 - else: - Area = width*height - Area1 = width1*height1 - Area2 = width2*height2 - ratio = Area*1./(Area1+Area2-Area) - return ratio - - -def main(args): - #facenet.store_revision_info(src_path, output_dir, ' '.join(sys.argv)) - dataset = face_image.get_dataset('lfw', args.input_dir) - print('dataset size', 'lfw', len(dataset)) - - print('Creating networks and loading parameters') - - with tf.Graph().as_default(): - #gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=args.gpu_memory_fraction) - #sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options, log_device_placement=False)) - sess = tf.Session() - with sess.as_default(): - pnet, rnet, onet = detect_face.create_mtcnn(sess, None) - - minsize = 20 - threshold = [0.6,0.7,0.9] - factor = 0.85 - - # Add a random key to the filename to allow alignment using multiple processes - #random_key = np.random.randint(0, high=99999) - #bounding_boxes_filename = os.path.join(output_dir, 'bounding_boxes_%05d.txt' % random_key) - #output_filename = os.path.join(output_dir, 'faceinsight_align_%s.lst' % args.name) - - if not os.path.exists(args.output_dir): - os.makedirs(args.output_dir) - - output_filename = os.path.join(args.output_dir, 'lst') - - - with open(output_filename, "w") as text_file: - nrof_images_total = 0 - nrof = np.zeros( (5,), dtype=np.int32) - for fimage in dataset: - if nrof_images_total%100==0: - print("Processing %d, (%s)" % (nrof_images_total, nrof)) - nrof_images_total += 1 - #if nrof_images_total<950000: - # continue - image_path = fimage.image_path - if not os.path.exists(image_path): - print('image not found (%s)'%image_path) - continue - filename = os.path.splitext(os.path.split(image_path)[1])[0] - #print(image_path) - try: - img = misc.imread(image_path) - except (IOError, ValueError, IndexError) as e: - errorMessage = '{}: {}'.format(image_path, e) - print(errorMessage) - else: - if img.ndim<2: - print('Unable to align "%s", img dim error' % image_path) - #text_file.write('%s\n' % (output_filename)) - continue - if img.ndim == 2: - img = to_rgb(img) - img = img[:,:,0:3] - _paths = fimage.image_path.split('/') - a,b = _paths[-2], _paths[-1] - target_dir = os.path.join(args.output_dir, a) - if not os.path.exists(target_dir): - os.makedirs(target_dir) - target_file = os.path.join(target_dir, b) - _minsize = minsize - _bbox = None - _landmark = None - bounding_boxes, points = detect_face.detect_face(img, _minsize, pnet, rnet, onet, threshold, factor) - nrof_faces = bounding_boxes.shape[0] - if nrof_faces>0: - det = bounding_boxes[:,0:4] - img_size = np.asarray(img.shape)[0:2] - bindex = 0 - if nrof_faces>1: - bounding_box_size = (det[:,2]-det[:,0])*(det[:,3]-det[:,1]) - img_center = img_size / 2 - offsets = np.vstack([ (det[:,0]+det[:,2])/2-img_center[1], (det[:,1]+det[:,3])/2-img_center[0] ]) - offset_dist_squared = np.sum(np.power(offsets,2.0),0) - bindex = np.argmax(bounding_box_size-offset_dist_squared*2.0) # some extra weight on the centering - _bbox = bounding_boxes[bindex, 0:4] - _landmark = points[:, bindex].reshape( (2,5) ).T - nrof[0]+=1 - else: - nrof[1]+=1 - warped = face_preprocess.preprocess(img, bbox=_bbox, landmark = _landmark, image_size=args.image_size) - bgr = warped[...,::-1] - #print(bgr.shape) - cv2.imwrite(target_file, bgr) - oline = '%d\t%s\t%d\n' % (1,target_file, int(fimage.classname)) - text_file.write(oline) - - -def parse_arguments(argv): - parser = argparse.ArgumentParser() - - parser.add_argument('--input-dir', type=str, help='Directory with unaligned images.') - parser.add_argument('--output-dir', type=str, help='Directory with aligned face thumbnails.') - parser.add_argument('--image-size', type=str, help='Image size (height, width) in pixels.', default='112,112') - #parser.add_argument('--margin', type=int, - # help='Margin for the crop around the bounding box (height, width) in pixels.', default=44) - return parser.parse_args(argv) - -if __name__ == '__main__': - main(parse_arguments(sys.argv[1:])) - - diff --git a/src/align/align_megaface.py b/src/align/align_megaface.py deleted file mode 100644 index c22da4e..0000000 --- a/src/align/align_megaface.py +++ /dev/null @@ -1,240 +0,0 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from scipy import misc -import sys -import os -import argparse -import tensorflow as tf -import numpy as np -#import facenet -import detect_face -import random -from time import sleep -sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'common')) -import face_image -from skimage import transform as trans -import cv2 - -def to_rgb(img): - w, h = img.shape - ret = np.empty((w, h, 3), dtype=np.uint8) - ret[:, :, 0] = ret[:, :, 1] = ret[:, :, 2] = img - return ret - - -def IOU(Reframe,GTframe): - x1 = Reframe[0]; - y1 = Reframe[1]; - width1 = Reframe[2]-Reframe[0]; - height1 = Reframe[3]-Reframe[1]; - - x2 = GTframe[0] - y2 = GTframe[1] - width2 = GTframe[2]-GTframe[0] - height2 = GTframe[3]-GTframe[1] - - endx = max(x1+width1,x2+width2) - startx = min(x1,x2) - width = width1+width2-(endx-startx) - - endy = max(y1+height1,y2+height2) - starty = min(y1,y2) - height = height1+height2-(endy-starty) - - if width <=0 or height <= 0: - ratio = 0 - else: - Area = width*height - Area1 = width1*height1 - Area2 = width2*height2 - ratio = Area*1./(Area1+Area2-Area) - return ratio - - -def main(args): - output_dir = os.path.expanduser(args.output_dir) - if not os.path.exists(output_dir): - os.makedirs(output_dir) - # Store some git revision info in a text file in the log directory - src_path,_ = os.path.split(os.path.realpath(__file__)) - #facenet.store_revision_info(src_path, output_dir, ' '.join(sys.argv)) - dataset = face_image.get_dataset(args.name, args.input_dir) - print('dataset size', args.name, len(dataset)) - - print('Creating networks and loading parameters') - - with tf.Graph().as_default(): - #gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=args.gpu_memory_fraction) - #sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options, log_device_placement=False)) - sess = tf.Session() - with sess.as_default(): - pnet, rnet, onet = detect_face.create_mtcnn(sess, None) - - minsize = 100 # minimum size of face - threshold = [ 0.6, 0.7, 0.7 ] # three steps's threshold - factor = 0.709 # scale factor - #image_size = [112,96] - image_size = [112,112] - src = np.array([ - [30.2946, 51.6963], - [65.5318, 51.5014], - [48.0252, 71.7366], - [33.5493, 92.3655], - [62.7299, 92.2041] ], dtype=np.float32 ) - - if image_size[1]==112: - src[:,0] += 8.0 - - # Add a random key to the filename to allow alignment using multiple processes - #random_key = np.random.randint(0, high=99999) - #bounding_boxes_filename = os.path.join(output_dir, 'bounding_boxes_%05d.txt' % random_key) - #output_filename = os.path.join(output_dir, 'faceinsight_align_%s.lst' % args.name) - if not os.path.exists(args.output_dir): - os.makedirs(args.output_dir) - - output_filename = os.path.join(args.output_dir, 'lst') - - - with open(output_filename, "w") as text_file: - nrof_images_total = 0 - nrof = np.zeros( (5,), dtype=np.int32) - for fimage in dataset: - if nrof_images_total%100==0: - print("Processing %d, (%s)" % (nrof_images_total, nrof)) - nrof_images_total += 1 - #if nrof_images_total<950000: - # continue - image_path = fimage.image_path - if not os.path.exists(image_path): - print('image not found (%s)'%image_path) - continue - filename = os.path.splitext(os.path.split(image_path)[1])[0] - #print(image_path) - try: - img = misc.imread(image_path) - except (IOError, ValueError, IndexError) as e: - errorMessage = '{}: {}'.format(image_path, e) - print(errorMessage) - else: - if img.ndim<2: - print('Unable to align "%s", img dim error' % image_path) - #text_file.write('%s\n' % (output_filename)) - continue - if img.ndim == 2: - img = to_rgb(img) - img = img[:,:,0:3] - _paths = fimage.image_path.split('/') - a,b,c = _paths[-3], _paths[-2], _paths[-1] - target_dir = os.path.join(args.output_dir, a, b) - if not os.path.exists(target_dir): - os.makedirs(target_dir) - target_file = os.path.join(target_dir, c) - warped = None - if fimage.landmark is not None: - dst = fimage.landmark.astype(np.float32) - - tform = trans.SimilarityTransform() - tform.estimate(dst, src[0:3,:]*1.5+image_size[0]*0.25) - M = tform.params[0:2,:] - warped0 = cv2.warpAffine(img,M,(image_size[1]*2,image_size[0]*2), borderValue = 0.0) - _minsize = image_size[0] - bounding_boxes, points = detect_face.detect_face(warped0, _minsize, pnet, rnet, onet, threshold, factor) - if bounding_boxes.shape[0]>0: - bindex = 0 - det = bounding_boxes[bindex,0:4] - #points need to be transpose, points = points.reshape( (5,2) ).transpose() - dst = points[:, bindex].reshape( (2,5) ).T - tform = trans.SimilarityTransform() - tform.estimate(dst, src) - M = tform.params[0:2,:] - warped = cv2.warpAffine(warped0,M,(image_size[1],image_size[0]), borderValue = 0.0) - nrof[0]+=1 - #assert fimage.bbox is not None - if warped is None and fimage.bbox is not None: - _minsize = img.shape[0]//4 - bounding_boxes, points = detect_face.detect_face(img, _minsize, pnet, rnet, onet, threshold, factor) - if bounding_boxes.shape[0]>0: - det = bounding_boxes[:,0:4] - bindex = -1 - index2 = [0.0, 0] - for i in xrange(det.shape[0]): - _det = det[i] - iou = IOU(fimage.bbox, _det) - if iou>index2[0]: - index2[0] = iou - index2[1] = i - if index2[0]>0.3: - bindex = index2[1] - if bindex>=0: - dst = points[:, bindex].reshape( (2,5) ).T - tform = trans.SimilarityTransform() - tform.estimate(dst, src) - M = tform.params[0:2,:] - warped = cv2.warpAffine(img,M,(image_size[1],image_size[0]), borderValue = 0.0) - nrof[1]+=1 - #print('1',target_file,index2[0]) - if warped is None and fimage.bbox is not None: - bb = fimage.bbox - #croped = img[bb[1]:bb[3],bb[0]:bb[2],:] - bounding_boxes, points = detect_face.detect_face_force(img, bb, pnet, rnet, onet) - assert bounding_boxes.shape[0]==1 - _box = bounding_boxes[0] - if _box[4]>=0.3: - dst = points[:, 0].reshape( (2,5) ).T - tform = trans.SimilarityTransform() - tform.estimate(dst, src) - M = tform.params[0:2,:] - warped = cv2.warpAffine(img,M,(image_size[1],image_size[0]), borderValue = 0.0) - nrof[2]+=1 - #print('2',target_file) - - if warped is None: - roi = np.zeros( (4,), dtype=np.int32) - roi[0] = int(img.shape[1]*0.06) - roi[1] = int(img.shape[0]*0.06) - roi[2] = img.shape[1]-roi[0] - roi[3] = img.shape[0]-roi[1] - if fimage.bbox is not None: - bb = fimage.bbox - h = bb[3]-bb[1] - w = bb[2]-bb[0] - x = bb[0] - y = bb[1] - #roi = np.copy(bb) - _w = int( (float(h)/image_size[0])*image_size[1] ) - x += (w-_w)//2 - #x = min( max(0,x), img.shape[1] ) - x = max(0,x) - xw = x+_w - xw = min(xw, img.shape[1]) - roi = np.array( (x, y, xw, y+h), dtype=np.int32) - nrof[3]+=1 - else: - nrof[4]+=1 - #print('3',bb,roi,img.shape) - #print('3',target_file) - warped = img[roi[1]:roi[3],roi[0]:roi[2],:] - #print(warped.shape) - warped = cv2.resize(warped, (image_size[1], image_size[0])) - bgr = warped[...,::-1] - cv2.imwrite(target_file, bgr) - oline = '%d\t%s\t%d\n' % (1,target_file, int(fimage.classname)) - text_file.write(oline) - - -def parse_arguments(argv): - parser = argparse.ArgumentParser() - - parser.add_argument('--input-dir', type=str, help='Directory with unaligned images.') - parser.add_argument('--name', type=str, help='dataset name, can be facescrub, megaface, webface, celeb.') - parser.add_argument('--output-dir', type=str, help='Directory with aligned face thumbnails.') - #parser.add_argument('--image_size', type=str, help='Image size (height, width) in pixels.', default='112,112') - #parser.add_argument('--margin', type=int, - # help='Margin for the crop around the bounding box (height, width) in pixels.', default=44) - return parser.parse_args(argv) - -if __name__ == '__main__': - main(parse_arguments(sys.argv[1:])) - diff --git a/src/align/det1.npy b/src/align/det1.npy deleted file mode 100644 index 7c05a2c5625e0f4e8c9f633b5ddef5e942b03032..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 27368 zcmZ6y2{cvV_dbry^At&$GbEKr<-Yrsq%x%eQBuYR4M;^JAyXn5N+g+5G?C)I`z4x` zN)w@h=2@vU*U#ts`>yr>eg5a(b=SJ*oOSkH&)x6Ydq2-}_RjWno8u!YGDBpuNl@^* zzz7ptJ(Df&CS&wWLe@q^M)X6_^(MbQAA~G0hgSEwkaOth#GNztV8-rIYTOPSCTz0wW^8YW8++d)= z_uxyNH z^03lql<>=*RL(mJsVQLs$~N6tZWh7%PkbvzB3j0U)04kG~m`ou>z^>F9j1_%LRH;-$>3O z1vXDbo0v~76Bv;1!ihuNY^9#Ru`P%QK#eeCSWtLDD0x1a-PYV83~8D`ju%?mzE?f~ zzw2D4@P97^1{p0vmA2{B=94I&-xtL;>D38dlHJ17my6l1S=vO_BUb2s?vG$&OC$Cq z3&Cq+ykPc_906J9ELeB~*-x=tLH5>MqBKvHJ+(C=p>K8y{;qv0IMtv?qGw*ie8<&f z^wV^K`0zw>EcmSOS6{KfG*OrA@i}fAFFa+tp=>zeU-!Vm^>S2fMV9c@qr*b|YYKv} zAaPQ0XSdLMQo7)PaWh$aP>CNny#&i5Pt|*A#|w96=hcrgoC=8_n*jL!^bzAn_hU3oQbh#bxzvkp6w^-6|J*9KJvJrd0C5~ z#CJ97kDi5bd)mkX|2|<$nlw>SJuRr%)-3c%vm~9~)x>z7BtOgaXw*UvLGqFJLPysm zp>=33i45?R3|Qg6E_hwI!hdCy;(yXfJYl(fvZr!c;YqSNZXB*O-h?BL&t#DS>;IqB zv4}gXtzJBCaR+tQ+UJ zO;9oJHd5wgucmO7s$4FbwUxd|il+ur@vNw{mM=OwkEh7J0hbU1yXf5~xOVqm?wLJ_ zR-ZY+zsN@O&)e?M*bG!O4VpK1FXnt^C+cQ|7;+r@tJgFJCCwWmH!1&dv9@Seh2izprK|)HpyJ zWxRRGIx~z6SpMHItNsUOvHyWtx>Quoc9@1l{eR$<_}`Up6|S~5T>bwK?$G}McZkan z7iAZ{L2#2jL}rPIY_&;*P4|u2A3hvo(;mP_O97RHM$i`7307Yv;7_+P-26A6_`Gi; zmThUcr+XWI_WVpve2}1ZPtx%2G!eiGX})%^7JjMD247hREbmoDjkyXm`MVVkpVZBI zXd#dLsLm^lx1w>va`L-+101=cNpD5nBomIw(*yT?A^Xk|;l8PHJZhwLL;I*R5IIzx zy6MH>Eu#-Caby-f@$>~cr%eMj?b-Ymjlt}T!+3j%J@3$p1Je)3@SRsG3r#!#nVO@y zgOdh-IZugByYYxKYbKD?`b6LAy3_YY>F6v!kESiHVX5u zE$e2Bb(es?wkPx)i9n0?Rgko`1|mr($aaKqXH2#p+#{$Sxg5p} zwPi+Af~iu!Aun8Y3NI~J1c~EwxM$5uDrk$P4_saf3%4Pc>PhI!YgfP{P>y4hOtl@d)be?=Fhen^I?f2*K)`9ab*v=3^= zDTCv7!WUm_NB7p}MAKD+C)AqK%zIXR=ha|dxP20czxp7We0@eF^=g%}z`DenFJn-nfc;1AEt z;L$UUF-+$anLJIFS}v2sPmgqXz6U8)oO|Rvy#cjGu7npJt>;`)R4E2l0}2fM~U{r zZt}P^5b%^Kc@m;Wz5I%~j7y~mokQ=TqDX8KsJGr&GH1>XDD{Y^alr*BRw_gP z{N0bW3*{Ti?WL&X&m7jUeZH+lRX%KJ5{Kj66#71HrjDB)@x#lm;QPYeTv9s~|FkW{ zSNeOvS@adU4#~$MR})~}eO;_O`iwl-cOI(0jK}Hv&Y-q>0%mK7;<<#&SZs9tpg zh32ox_7Ot3_}-B`ds8T|{BRN;S?AH-_Dclfty6=S zCHt}2E|4zVvXqJ*zb=&jqEDvfnQ?<^a~?Bw8I12$VpfM^AhggK>z{nWU-m`pTWbe; z=U2i$iyEx|P=G65hB1MW6L?te2BDP_S#|piwg;r*-qD6UDef3r$Sh@^KUDbf&JsSn zI|Jth8`9>3s_@}*IVNtf$K3Lxa6jLZruUv=Z@Y$5&r78cC-H%u+rE}AzTblp>)t?* z_6-(NF&cIHwxdpzH0?fj7LS%na`q#I8RX1>{`!76yks1FlGuwQglg!#$&|-Ekmi!k z$mFhM@>EH6zCPs_pZZ}k|L|cY-7Viu0(Koo6}*RSGin6i-BoCdRWZh;J_WbVW{hrC z!kK%jAa3taR19q;y~__{OOrM05L-EPZ&=vegb}Vm$dGz3`3i8``xp0zNGnp+FL2nR$ zEX_VjJO#bvqqibiB{v+)*Iob*-BuVlcn6fm)Z*|(_GsZ`i`jm)L@ig28b3;|cSxVe z*^!|_o0qE8;%hoO_$CRx`qv6}iq){V5;-n%a3B2GF^1RFnDV2o5BWuLTR!%(7^dG! z$A~-&h*tZDtvfb>(t{328F_$hQT3oB+lTUFPsOO=qdyo@<-^_PS@GHmAxg4-(7K?> zXK+vK*_jPf-_C#$tAcQ5rY@Vi{3O>|Fq4m4Awk=Vp0l?{>LJuUo7Op>Mz_*ef|D;x zV9|B~3bPIV$W;DI-WXOFzW$25_K4$DL!Om8QK>wT& zm|&`a9<}Pc>Y_F&8f<)w&Zk3B%5eyH(Z=*qBl*<7chG%&xA4|PJ8<|_4d;E#__c4% zEW39rRvLeTGCr4Kjxkr4H>BCz37&YX@loB^Q2#|b*6PJVwUaA)JyC<<-o{Y*LZTqvc_k7eWdBX7~K?h0-T z%EN|B_O>++j9m8^!Q2;(q*>j1_}93NJf15{&m=FQT59jmXX_~Hhqp*qKpBy?djWIb zB1|)ug1z_BaJ!Z)m)!jrte)ktuArfG*`ZTty&{GznrBS?;+H^0)hRHKyG0K4d%=x> ztpa~ZZM>g!lJ#$D#_MN7G1NqbZ#W3JO`;0*)GYYF2u`u2CrhzAEm3QR%X8!2o@lN2X^c@1uz7$Bu z+M$RDhlgo+h#_;m|25F-&ou%A!MT^YhdwbXQn-)i;N$?6CTlgL4&3_c^>l$ z+xDEr@g4GDV$Ru>f~6gQ$PJ7VuaxxYSdI zEr;#t^=TWZ!o*{2^=c8iTJ#az(EkZT>J!0HgiF}%>88YHVA0J4`UzrHCfy59jmZH}d(X zW4OF zYha%DL4S@CJlYh9RkG(v^YKdTvrebMHpZAAexAtxyUQlboh2l=11w)Y0)tr-?< zkb5zby3Aj|gO@3y-lXq>;WwiNX=8`M?UiF_K*v|m*)p8=dL5+8W;oYR{XCT>EVHJ@ z(~Ricff6{mYB;X39z{PmC!zneTwI#b$xhUb;|oVThY{VEn9Am5Lb=U;Wa79Sd{JCY z7ZmB>^()G>DKj6xslCNI^JMs2wi@G}se#J7a*W^l7wxw1fHH?K!g7OBJj*tL;lJ~E zx6cB8n3P~(>LTzmswCr!4A4(94!2#GV2)v_U~ha8h^Z=#J*x$7HeK+rdluciAqg+L zEXS_Lq2$rDTNqva5M8BD@IQa15z8J=?sM-8etTGoQA6|U_q{j)ciQveMUe^lwDO$5 zz}|K6zv}__-6N?|;c3`$*PSgLu+1^j-jfq{1qQTD9_UlJa|y?u6J z-V!}rE;Yd9)chb=J{MmC%?{4mVobz-EmXzqoBF1|$qepE(+Is?jSfzcYkn zJu1R3nuhn5`rxjF;cV#Lr>J&Gn#=CAn+=ulW#|CCZM8lVR+&%)yKd1ABmMh_?%`u{QbQ(L{xeq^x zi}KeK)8J=(nJ{-*I*wfUjEu@wG17%4#*gBnb{qMII5|+uea51PHLy{$uHbIv;WQVvkTQc-3_tHhNBm6# z)tf*|%pGxvXbD@lG@XqJF@_D=J#csPL+Br4j^gRApbK@~O@tX|4nJI-IUf#mgE8~dOi+Cmok)Eox4%jnBst>&w{QcL#>Pwd78{yXm_7 z#V~*ep~-S8#{7HEZ?63Sy2FRlUAGXUGDSwjtY>mA%V_N2&AW8 z@4yj_rL?2x5ZkFzfhxIr^w%p*n0hc=*yXc|TSXoKYiW5Z>7EWpU+3|4ub;yxryaC> zWD=P?Ob2CC(plqBQ(kwGf#XF-!bkjphRmb5;IS8cc9rE9qs#`^)?D;E`<0vyH-V|C zg%~rk0gt2&=RF2qFn{+MIFzUb`rpklrBR6nw7cPwLuyp;S50UpW=YTGoUBiSF;RjYZ;4wo-;hW5Q+O0Ytir1Aw%t2E$sw@+{@NndF!(*VnFcNAnRthrAzOd)Y zT6|kUE+&58PdvniQ9=LU446KRo+;izB_nCH7+Ep4D~d+UIRgiOml3hf zO7P!oOr!rypy!S{!JE5($yMCTFN9^#FrPcH&1C~k-&DoRyDqa`$q%XKtWmV1%1!tu zVt`CK+KgW^2<`E7qUupV-4)bfzHlawd>Je_|KJy1>wH3OmOsEo-M2(nb{mb^Cc=YV zTd_rM2T4rmfm6?%c}?XQUVZW~E>bYZ=wD5(4*}{M)g(+b%sIzPVbe{=_OU|bJ?D{6SkhmXLby~rDtq`8&lYg|SPQnsYa}wvEkxBOCy}|Gwo@9nw5ntmt z1?Gnj;`>BW;MB5kCl4`60&XCgWn|b6C@|f_uM^;?;I#WN(QfU(r9FDn-3uc_wl^ z#%?*;IwS$U&)y^u<9jhnYY+M7Q_eJWk^6`zvC?KSZkF?dC~Rp1y>GFowJ-+_dL7`Q zLJ&2&od_ef+W5>1JFrJ|fE;jcV@iEilvNysXI0kL=LbL<;Za40fjbzrl&fn}|GV<6@*}Nq9T)F{>e29+~{WhB<7HdI*nih{tJh!{A7rC7%?P zEcozF7dBk60d-*kbC4*+O_|1YEsr9e@gixI|=%<@GTWs#j=U#XOVST zUKr#TEv#(H!08Xvs9CTCEPmPtYTJj=28m;2gq=N?oD#glIxb-@%v}tu*tsY$@&O)Hs_tYx}oou zBl;S9;+oh5$UeCkSl>AQ=gSU$v$ByLb38>o?j9ohnzLZ_gU9&j%3_TBTFHDnq^b9- zY+P{90Y{aH+T|W-g!5AdXFu=9Le*wXs-w7)>oxv_EF&*^{G%ebx|R)B=6F+?eNklI zmi=hhJciq^v4p+(4t)PL33_PDF|OSnfb%~T3ys4w@zJqE)V(DU?EhB6jFrGg4=jVa z@vGU|R72KfnG60h%VDhiAn#H2hRo_MrMmKaP;Q+K5JzO^rh1{}yeVitaTKgJv*4;h zb$o)UH7?5OBbJ#0CTe{k%W!v)H0^Gzm|K z!1`6}w5<`1m?@yw9(TjIsu*ZAI6jE~KG03y&(z<1Wy6=gCN~tqq0(;$s{M$CaqTfY zMy!d9Uq2KCBAbM5wo6I6`XxA-T#IY!#KcR2#JcwN<02#pnxc z_3bn~a8{SSQNBeClxBnRfD~zWDi(}cVnYM}>e8L!LBvIvNa7NPFs%unAop@VYM)<$ zR<)BcsM>-qw3Fc<_dEiLJ(Sx%JH>WvYQN#~R$Oz+wdocFT@ z#-13$H|`l{*HJngjRV7=xLpLzCT_!qBQA8x$9O8y7e=o?tAi|lkySW0u%{W4+(tnK z=d||YuZ@4v*nR~_cBR(4?WI_)b`UIAH{#;kX|$rwlAT#?fwzY_@`pDH==}}$WWAX; zm`O!2KUaI+_;EGugrDq8_9e1@!#{d8=ocL?F_oJsS3*mO2L$9zg|?_5c45&ZJQ3)^ zXO5QOB<2`5o{))o4MVwJZZK8J*#$ZkiQLdymw&yWJXr5uz=g~_w6&&uo7fyK-z5${ z=YON=pk87+r5P0#6~G*q4(MP0MM(4pP^5M@EAA-Zs(z<&NSiyl%|1x%j~S5Zr^-m# zCrxOp^+0bq%Ht(pa%rP=SP(y-w<jYt$Tc)H z8N&mfo8WGZP0;!J2(G0OAin$pb`)NMC$UP{`GE<roKg5Sri5X}>4UR^IM5tOj+x$o;8>@LHy`at~kK35P*&%lnS)4*qn1I{Zv z#zuS?MwVsz&<82r5a88~JEvU{M84A_g+v|{GBmku$X7hr;{Xv_1w?dCGn@JTnE(fc zWkU@I5ahS9VVz~@@23FygT2j&SXI!VuH@^kF}BO1En!GN7+mXo1io9ecwUq~{^>zt z_p<@+d<}>0ywNOH(~f~CFw{Q*%})9?T<>(7BZ<8)wL z>=U@I05D^}1-Tf17tJL!xNXZhOtbdkrhUCAx#vF7dMAOfyBJOK8*$b0Q<$|Z2~SI$ zV(YGbua~jYKz8RLDcVfCefdZu`m6}5%IFHtvi4nD!S-BHZ(%qCGO6&SM6 zl8Yy7MMxYMx%~p+y}zr# z<8W=dS4T7cEPgL9Wm6dnI{7ymR>jE^ll4?*{o$t%-6ke#>_r<%V8 z)!DIZAIS5h#eXqngOIWX@$71F1$z2TvrXI;#d>5d*xIuf@%mmbjM^mGVEAvZKua_f z_5MA^!{sCSXHP$5aU)4^-Y(XhXav36AL8FT6Ie>F6i8)>(-(FldD8YK*yg7}w}g(R zKfc&=SFOV=AvFzeml(hmb7wr!bdFV47NbkpEKt1K$D%EOFPlCAUtC|rq_rckC-pvA zGTIW#xdt~KzLKlOoJS>z44M~}25U$E#Nr{Z@S|ZF-EmQaTvW@#%+h60apn*lX}JNJ z$E%@v`z9PO76a^qq2N(YG5qjIz+LB;vb{bgOlja9D_`Nt|4z=v(r-d^3U^?4+yd}? z+e9+lMgsbz>+pq{DBfK$A3s}((Fb!nTbR<$cE5R;-m*Q|ut zn*qFWtt$V})XTOw3WQamH`%4#HKd~bG2XD-Pt3PC!GusH&>if-RE|H!=)+^E?fsW{ z=~_G7J|Rc^B6c&sm%c2hP78!;E6J2`etdszH%fXpp;&7U!2Zcx$IOO5vyX!-Qu>&A zcr=;t>>#f1`7E6Bc_`mIt{t6qk-8^;yzg(^yiAUgqJ`M7Rf@dYc~LNq2Ef)2Dez{fIln3>L+foF7$SR|9KG@j z*9dmfoXp8wUV4*Y<-0~f`wLh8u~3idi5&#Zqp3vknSk2-i-j$-o`QJDPgZ`mQeflT zg2CRIkfNzY_kS1Q?6-}enUVxQ5@c!A%a>5c)dUZ3O@Q}a3?yT+aroLpu=*XtpV7y` zUOgBC76!xEsmIZ!R+K+#kEg?uv~laUAadw4V5mTc|9Xvd8c8PkW4nm`giTaW;~_Cw zNCxY_mmuK72dEAW;xFXlXv4+RKu=$V8_H^2`;s3oy;%tBn(biTabHr=atgmajmDw1 ze^~#!aWF#cK0EO`ooAL+60f{>j65}^^RM~By>J5wjjKZR`Z<-aZ~Y8A zBDdi1CUN$|-VMdyPT^sxqp`{OA{)`}kA01|Ft(x?&1+cw%{EPZ4|Bmr`!%|aSqAFW z)A9BtRm9e4wszEIxL4XiqRl?Dv7$TSX6I%&-*+4$=u5%E`wLL9a^r+@Ov*0;9ATipBhIBe&%&zWYtDC;`tcNdMXVwHJjP(lEZ`Er3YXt_(7!1 z3@9IM5^})WN#gdvIkz{3SQ($ph00c!2 z;MW01R4a-X-riOtuvbw;&-b}#BAdqUTdg1$rJmyP`umt%Bqf+;YJ|NlkJx3pgzH|t zf!T?MygE_>w=UQWmiwh4`BgLO6BJ&vGlfl8Bd6igAv&1oJ(IKsDDttJMsf3XN8!v24PJRfg^nAh%BvdN!91DJfNn3G z?_M?NQJ796$CsmNSsS01?1#D5NoW)$MTgk8z;8=UTD`&&u1u=N&sPgjF7P<4c916F zZ$_Z@P!1|?LcIR97#&;2v(Fm~iRAVU)VZ4ulkEba($o;u)ZM7f$sstq_5(H9Ed$fn z>}OYET-lizGrZ8U3NKjvV&_vN__L6G!uu`?yr|&?33QC*7D6dZTU-rS{tf!FtWCMq zY#E-lq=r44x)k>5rSN{Sy{u^DHndR}p(^Jj_$6&e?z`?S8KzSwAho~Pt*&6)Z@eFD zXWxO7Z|dMqsQ_r*4fy-v1^OE8f;M>*`fJ)3fyS>t5MJ~ZXAOCQasE=M_2mk#&1_|g zLqD*-OWBb2V8h^?z8-RG`iS#(aq3%rlW>X6@IA2+w?#;B`Dd@lw|Q+)tNQ|62lYFN zOg}7{7l(%x3h+~vIx}D3g+F6=LubZ0T>K`Mc$n`KHeH;CO%>Pi!Pv*>e6JYC>cr4B zDPtg1Js7IJXK>!_2e(`A;NsJTxTf;mZf6Pc0`h7}+Ny`>tQpH)^@oN>o_V)m+opu*;?!JPQ z9R={|=>$@@N}adecmopJ63ne2oVM!t@V@u5EHLaOxOulTd$k~bxPJu(r5|B_XLR{; z<6N}OzKz?Cz9(WAvhj)74ZLwR3uC{AV#Xg!{8hIWYY&v+c>e|-y3c?;$qgH<|0%*x z+EO%W_9L<_O@x2!kKht3|6=jxc_=kAk+-fk=ZjZ;hO(@k+&fJlmp<9U``ZuV{U`CX zPcx0Ce!R(EZ#cz*U-e=KD4+-d`*VX zJ?A$JYPB9#wUs5WOeIwX0{LqNVUmc*_1g&*V9`Ei*lJaNVgl}t;a!bF}wo-%KhSN4DqVWrir+4&hVm_Q__L^Np6DFMt3`Uq)yHG;yK*l`W?q5Un)keLk^}7dM6Zk#;ZKK=j1+? zFk?N75^sR^6UCsh!v&pR?10%PXYxZ1D(E-o6J~eX(c5|2RJ_<3M3?7N8EtiZILQHu z%EUl-p&vh}eG8g`X3)hO#-nzHGgrVL7<#A<8&;km(z5+v-BAQT&Yz^}sXyS?t!Z>% z{Z|x5ZCw!himYv_f8b!0-;*PtqEH?c)`P#mbOynI%&-Ic8 z?{jdwXB&%?eTe-jd+YZ;iiE;#Zz1`BGnbe$28OL^#L0#k#M4`b%$@Ee*wAmnr&svk zYTsy_T`oX{6_>~a5oap@Z9W7%o=*MRd)WBGLB6isgr-~GgJQ;Mz&)y`=8Py5(L23N8@=W{@5eT_0 zo1pEP=%9u?3S17HhKsT~+#v8c`*I=^C%%<}0Q)!W+-^^NzH1nN@TZtWPfrj^#;s+S zc1rS>rBU4fZVyY=6XQkmtnu&m3AlatEn&6#3ABDXfLHa^$(6tfaB9d0e6+k6zrLGlu z@MCHH;~*zCr)nua)k;S59b350nw>(wRr-8i$4Hu2@D#1hckz$=WQFJb5qvzupuQ** zrcD?i;}&M)nIkGpYIi0+Yx)ftmNHyzZUWvdxj?3cok1mC;MUTmcrR!L8>=RP8`Si0 z>EpBbVQMK$2uOl|8JXIoJq#^u}Qw^K0xfJ!j^}&~K0xXJm;PnRCXmoEH#0S~% zLltX@f6Q;zvSZN4ArZ`On!U!=a+3J(+bUs6y$gHR;|<~M!(dbX1?JSX4wv|}v&Vn( z$d_k(P<49_Q<;5vQ2*FLZRd%T5k)H0`*u41_R10dZjiwvdk5K8mq{>p$`huUI+Cv` zc!&dAMp283RKCHX2fqb23mrx|u_T+3T+(3&G+$w8beY4fh+5b+$mjeEk>bvFJAo9n z3$InW!O&OGf*c#CVc;77{rqGoKYk+>WV?G&)6bq?oD@3fU$4NAW<%_B z&xumsHM>~$&?I_Gu^eTNpGK#nrF`yL5ngzu7*ECT5Xyd)QgM6u*}LPpo$UdT zxT{1n{;uTGvMS{EsNd}3sv|JcY#aVvP5JdbzI3_jE>dg9(D%hZn2dp zyy3-Xygh``nn`@I$UfF^J{Hf(EMmsB9H*^Sz`n&9DB&6eVy6>uPwWuBW}tw7IPe%G zJ-VT3f-UVeawe-or=Xda7T>((IP^^`7CNQO=Z_ccq|g0Az`Ls*4o|n{hqm29*@cQ+ zL3<4Sc0db?_PW#cW4hSf$sKg<3{RFL^x}sm4a05P9^6oKe*M-SQJg+mR-i-`xcqU3 zrlLQv;p96qda)LD*gll`o-Y%w$Zvpif*kz2N*rfiSH@kj69xL4(%|17P3mrW2Gn1j z!l61tXmpqT-FMOBaA4hJ+?4zw%b#gTSI3){~|4HJH=D&pbGhRUJ_FTHfdb2S0RupK@b}>tBp`1)-L^S_zI)OrTy|tWj>^+UX|)s*6rwojUkO5(lmxr4Cc>@h8uVT7 zdip%F4jx@DWs8MrB=z7H{yJQauG7~C`|E8WSW^I2FMA;4Lkec}q*6WgHV7TOLFx3n za`6^b{M=OwD!J*j=yMh=EA_@>Z*u5!%gs2j!Bvu{P_`figMUfTKUTU_E^s3|HG3z?bW-Byb`&s)l@D;OojKmx+X%BXZ`r;&tqYCP zd3^UfFJ4#hjt?uFN%MPU=p;W6w*OWgehl-no1itBKmGOqQX~d_^zy%O@0d*bYf3AG zzmcJGHwSypY)9P5U!r`S2pEl27S^461`j_7>?UkA;2*8pLELi$EL^aW%~<2a?yt3g z4LBC=Xmmhh{z@Dt`5YH`C4on~8h0OGgG2Lba7N`sRO|l57EgRfKA$Y%`#z1ujvu~s zw83Kj&2%?jy4-_mgY)%dM{(X6{hGKX=74amGH;E`!Qsub1j2<|1$NWj1vjfEVEOzm za!Gn0E)Kj5@)s20r^7;0og9JDa=hOEpf9_4O&-?U?g6vn^DyW2Win8t$qyzhMZuLF zXyp}%vR~uqo-Z7#4lQ9ae~&`M$j>Zctv~n%2GEji4shX0n{dz9{W$Gl0xnmrBu@k@ z@YkS^bxYm>z9stwc7}PuPrn{enLe6lT$};=;T`OygB?^eGqD1 zPZGyS@-B;V%qU(D1|{Qoil#bCIy#1)Kh%md8q`?Xie1pZzL;u|1FMF>QsrmmZQJy_d?^ zIC7a0k8=Nm7RCv@#lHYzk3DAX*Gl2p;Tm|B^n$FqI*d;~k;zgbN=WuHeGGSe2wRuy z;J4P(#B|&fjPK5YU0+pTn$0J$1r1N z1@7$oMn0t9VSh)&gIvfLa62f=jV+sSw%7*t{%;ASdpDBDxyx{tM-fU6`Z2#PnvX)4 ze&ORwZBS@5o#&;BbGs@Hu39g~qNUsM*9|AOc=0)|DmQ}LseJ@TD^I>&wFm1C_|m-# z@1RuMI(WHs6MB6fMZdmMJ@_DQJ*z9?>{6+X-ymj>$YyK-uCw%e( zAF-c;fabCEVpA;bAJpl7e3-{`{}y9zsR~`}ssN*eiELZ)8z#XR@6Qv$K*$n2TKrOI z{P`h9j`@fgi`($+M|IkL+7T|b#}9H{3xvnCGeCx3g=S-O47Q4(rnCSqd@F>4HFJ2b zVmvzZjzcS%c=~LR1HZIB5uImCqmtQK3~n;unvybbtjmNRiapGNdb5c-*#Svy?M$D< zP5haZAjT%Tq}o&W<84(la*!A!3T17I3=O??f+%cQyI*HsviyN@sKSj{|D z-lH(BWXeIW9Knr_L2iG;2u@GCF!vif@X4|LAph1M+!|Bydg%gT<-TD&bE^R~bp?Uo zPdu4YoKGel?h;5XJdRzip-Remwu?UGg||5@;!JL65ad@!ZCd(EFxDP;fE} z1-rs|O8+WY6=BZBzb}Taa%q}q(~PsiLRnvVHB1djfQ;LHq&HkkFgolfOuF|1LLJND zcuz4tE4C0cX_`U#d@Ed)>_#h%3}HfJBCJj;gWj*#@T`?JU6CzGTW&RwM{N$EIme7P zS`R+=aq0*x{J9rnB8T(-s&p(KT(>Qyljx-Qy)d>wm0lIMq8nBz(GjmI;A`qQp5}7` zu74?lp*!wEsNN-Fps<<$J!FOUx9);Rp&41f=s2_u%%D>qR6u;~9{f?Ag-aX@*aQ0* zt|Gc+@OeNxI@GV6h>yR3sWA&_;?1bRUhDu~Y#POX{EFsl6rv!ib_T!GuEIx}20*8n zE_@v}72|cnAUekt&qn>hTi@=IOJeFYw8Ii=I`wF$damsX-P>UHPk}CS9WC@^jclgE zTL^jnlbm%`#QWB_V2IT^Y7xna&v->@5Uq&@^`599sy*nvv%;_q!AOJl;gS`HNb=ES zxTj?Wd*+WrT5kr?Q46Sy$!#*vya>z!&B$K2S@_pAh0J*E4#Q?t@^`h$-0JyuRDIFKUYK8jomhtE ziQ>#YvXX988BTk{${<7aF5Y;fMyI9^-nHkdqlo7{*b(H(uW!46)dn?!P{;dlZ~8lw zGTK6YgtlyzdNg%Vc?T8WKf&``M%4Je9rl0w4l3K05#6AfO!Txm-HFxGzAeA&5Dvc_sNQ8!ct(_r?lB7tX!4N`4nTqB?X%I~`XDH!Y z68h|GZDUcCkfF>Xp(0e~XW#ehd7X2gbN+#A?;rMF>ss$=DRSClais6nQ~K7~ zmYK6ZhVEXdi)AD1NtNGn;`r7W<8GhkhFo9H7@w4-4OjEYHz`1ch50mftSXTap$;ccV+4$$V5u3P$`STLql8()So?(ov}W z_N|bv)$%0QegzWKNlj#Q%_cf@Lmkl-c&?N&;hg2~QrdOXgsfX)NAp*o<95u~rc1sb zBD$GlahP5u{TOzT8A#ks-lY~$rKfX2>ZKy%f7+Jj$I2j|I1=A~X{LS8@6wOEGRU*I zUv%$N3%aZ2HCf)ML@%r^qCpZXh)3W)l6f?e2Hq|wn_d5kCV>-uqvJzEI)_sE%d?m% zq1~L+TXBe^mW==4Mea>e1YM{q=#fef(!0tM_-cM3iD%CeiG{DYqPLmcE~sN%J_h52 z+8x+*wU6^^QKFNZ7UAgGW64l$393~l57t*-lhx}D7*<&p7S8Mt?T@+6&@r0ut@9?4 zt&b$^=3JtkbC%oQe1YCL7f7xMK1v0nz6>>Y*I2{a8rd{N~y;hel)0?af4To*Ad-lYnAM*NDulWn|QoZ0hTr zN!+rd$g`bGnKLc`Vz;*8>FQB%=)n~_dD|Jr(aBh>rDFJ_bHUcnbr zLh5@`sH%G)jgN_=hwQ{Sr)7=w*{v#Wu4Ra(jBP+?c6c>S$SxCD-M$nhDT1A&HE8xYO zjnvR;7E`7$8P|G#BNDTt>DAvlbk8>fx?2U5$VdieJ10W(i)0U5ic5-xlYyjb~5Af{pxb>;^-g`70HuEUE? zOOQsK`;Pk@Aw^ft&cxv1+i>K&jnus<2d&I6G9eBIz%BVhs{1BkLt8V|EPO&7KRYu+ z48Jln4t*2q>210xv0Ze&_yIlAoP>D4f}w&U(_|b&w_eDi<%hOlp5g;0 zOwiul6#fA}??uFaz#cAo8VWOgEz#3^PE*?2=m^%4rrDdInU6YEy|$D-Tez5ueC7!X zz%;0<&vtq>rtyGn<}Ps68j&2MQP@SOuv66)B8__mA{51j`3MZtFt|6 z$ptAg+|z_CS8ybm-E*)u{T;K|#~$XyGeqmqT3YQ|DVqP@7=w>TaQO$Xap%RCk#jD( zutrr9llK;J=8pMTr;`hvwz@RkI)$7RwAv1~nOwVflIW`GDf(mC63{Lhf)<||h~+TK zc`Zs{(C!r-_HzT|oD0IkGST4nE{zT>G3LL<9m7#-%49|K0{G=#$7w}nlJ!s3sGX(` z#<+#Rs$K;wK^wUI&K(QBuOu4ZVzAruAzgnw4c*Mjxk|0GG(Tt`_3?T{H6`bfH(HbE z&uSmynDUF0>jH9E&4YZYkEgOG7O3gwt2ByswU=-{ z8N2ATEN_~1y_PvV+*Qzt4uiL@mT>&5II`DfK**0=yb)$f8{P+indKNzn4`$4r76;KWNGhLaY~5>gm>z$s3{N#L z(LkAG6qWB~g7E?!ckdz5nm!$G{TN411lbecGh0YwJ1;UAB1wwk>q!aGr{XJfnBjVx zV10xM^sFrdyh~w85Ka7aJL-X3J;B@a6)H}2rx{cjYv3dn_ z)#4(w&p1JzCAD(Lx|_*{nl3J_aVMOUPlRO;)1jh%qM&(+#j~45)F~ndznczWS1Hvo zlN<8Mj1kSGOTU25cVA1-&v-(0|9BFwU9Mab>Vs0`J!*972h-AHi1z}vpxhs4H2m%X zx1y7X)2#zsd7cmP>9|b?9*v|LujkS{DM2@XVI}ALRhR6w6UQ^>-cV=Tp~%1T;kMk! z;zIrQk|(_$iNlCM z#y2jCtnHdk$~PELlQSDh{M};u<;WTO_U9<9TRN7ecEw^^VuG_m9eF&MOr9vX(8#ELxbJ2TQSzqT z6QSqcohr;Mak|);Ar5-2gXDXcHhKMgU#-XJQto8mJlyrx9~{O?qmfrJ9T>Et^PC%) z9X-+Hi9G?isWtSLu^C-(>;|n1=WB1M#&W06o58M4+^cyKBhT)K<$z<;E3uN&aH~KO707h`j=~BT@ z8Z>k)wz7;mcXf*#dKV#5{Bg}(T@8!XW}w?w zFJZVm$-JQ#Xn~;bWqfcA-3aF1jbhDGt$`7WlHd?>*R7nY}ESmSDq#6}aNadiLSVF>HgU9V>j#1Vka7>x6qctSa@Tffa}~pP#ulcP>?k$+AxKZYxyByYk`|GSk_eksla=MMg*JSTf)JpYBQ&Ej+v2i#-`yh^bw2^H5B@zVbWBBfc25iHj%UCE= zN?*Lm1d*-+`~2DwoUI;&x6EEc@WXqI*?Ylwn-#A8M90X za_V0Psrx%IF4B*>sLG+18fUZN&miOHJQbf`H)5x%n$0t%b^y#d!+>#1TKY0InF+k=j~0 zFuN3vaTR%3_|^!;31ZM8+6PvDPMWkh%Hk>~cjVDKpW-qn&!tKh-*dBWbT{fqq;-ggB z(U=LY@Aa62|Gea$UKj3+%VKcOm{X{oX2&P;3fMXJGLcz31gBmn7}X^KpTwLoxZ4z} zmOlmirfA$VITGq7&tr`~cTu#byr&=2eLz zb}jlvw$?O~h6hPFLW`23-uI+9RspJ?OyXLl?I5V~8vNN($6f1fN9lh$;hlpd{Fpx- z-L6Ye&2VQ_d*p_3tA&s!=PB5wlMS~%Q~EkzlHG819$Azk!as|Dk{1hHx$3ZVE`N#+ z|9!I*U#zm94r`7e$Hgn)WY|{ra&Zqiepv-9!>=>t^Y#nh)O^vHDOu$A@l0a+atnEA zQ9_$W7-H@B#hhvNM$k6j#XWY}$xShrn|fm z{VTv@?``_kwU3GV5CFGloAHm_597AXEvUN00B-vx!2Tvd1nQN_%s+kEe+s0<1(pSCYa`z4r04@Hr^Y11GCJHMYEH~&?~FoSeJYYhqHmn zDB(N}Px(jStXyB>+z>$Qv{m`#6$*?)p9QC}GmAFlujcuND4hC869SL7GPU;hq-;SN z89Fo$O&YUd%Hj~NVelB%-tvZ4_k9?l9112gf70t&X;5)L7XmxyV3pP?!XL@Q>P?kk z`6!1Zz50aZUwp}@VSPl=Zy4OqoP&#OU%*UxJKp;3U5N4cB-&!6!R~1v&;OlvoBbgS zUIl(){EkzM&Dv+_I0Pj@r%DGcN8KTjjXfk`yb7D6D+isQuYz&7FQ_!7(#& z3udKYs0_@c-Od+q>EZ+$uv83F1+doV>}T8)Xu*6K(g$bG)X)L1Q(WzgC^#E(mP$pJ zl7oiBX!V*F(sAVl`4hDpJbzWO0sYF&qRTv*=!tl zM}o}>xJIJmW8uiyXfREi2vG8gF7+%X4u@pu*}s}t_;e>x87D?`R*!`bGd!`xyhe0( zZa7TyPNWrD^Xcln>BKeS0cm;KLM*%!xrtF4R598JEXo(aFp>y?fe)$Df;ev7DnA?? zIf7*T(4<4xlyfRJltvA!5&3D9a#jJVqO~SPFyDU+J$So~X)Cz{D{sx>9o+;&$kwYw zwn1lSchvUeqr6G{rumWGZHqeD9k$Ybl4w=3FkRv6{)`=bC#IpzB zIJ1uUd{E_oXNFQwV-+m%&qJqSuect$mGEI@2`P$|VI}OZlI^DgAZAVp$T(4$;Z}kg z--56*l4#0ut}j21cA zUx9bcGssDSXb({0h`fF_)O`#?yUk71Wp_RK+5MZ&`g}yxx2>0xU0sO-+pWm?br$@m zeY0Uzu@ZPQjkt%8hq*ad$%`6qB4%#MeiwSG*r-HM-@X@Cy^6$xd@DUPcnd-s^;pL_ zX*k(Gj`u#91-eDL{LXJv@ljbkvFy_14_DlwMlMtF<7(mk-{hlgVmekR72s6q^XOHw z4^~hMbUE|8)@t`hJYHZ2caxpD-7YOmc4Gz6ul2_Py+ZD~gEIFs68Pl)y}Y|1G$_1a zfMUjS-F98@T*889doiBdI|oet(&*CD zrJ_DlCCq(v1zvyE$6amm?05ZJw5z@nm2C1!NW@p0=j}cqYvw@Q{zTjSQ}P&gr09{W zFHCqx>0?B5AeFl8&jr&99M0DH!8NL?@DnE};Q_%cQoK){f75fB7EIK|&s#O|;K+R_ z6E_z8s}|u~mq;jIe}ic{kP4$sW3XhxOz_gq$Dx+TVCBm|_`PTq++5jCt=-DVK#wJq zUljUov>SI^?HPAEeLv{UO~$mFb70Am_e}oqEHbbeaK2a#S<)xTPaI)S#qV~~^%_@s z`=s?mcT+lf#9qMS{teXaR-r%&3IqO|BP2wp3YjcV-1ciW^q(pupANsED<*v69vn!5 z5KqAF%t$KdmMGet_kqa|kA>vfo%Eq^7J3zoB+6DVnb1+iM5V|W3{A$u&GSdl+e8b9 z>0$WCK@uCLl)^jLIcQa2KuL)lSfyB_Ysq1n({zg@HrRmY=6QGdc8)T^_b0^JSZ!5!bSD_6zmucdAB^#le?GbSOdk_YJ%h*(ab&v0 zagmcyPtUA!p^nR+!K-_!7TeSuq3%i)WaJ4Z*|;p?m{vz73TBr`m)ADTxMuol_Hv@u?TM~S|I)X64#2H{ z0;sx57?FKkPuH}J!iZOoiI-YCz1Z48T+#+HyF-i@RXwu)q;iE(ujdHUdXG31GiZ2e z46F;k!PJ<1A$xO$aGcH7d{<~BNUoQG;I2+|x2OQq4KwlH<|ioHo`Z>fOQ0=MiD9)r z(o++Yz~X>CG=zVJ*E+sr#z#w-F^fUXgc9gz{z??a--jYIAyA9m!?)Mf;G!>*a4P?c z=u_1ZSXj897Xl&a)4w^Ob>}nK6*hxmSr%>)?!>>hC2>QAfTEIJ(ZYSD2tF8U@F`&$ z{Pwor)PL_fa%3h8<2)we-hdIX=-hZroFz*vMvL>3SLCtk?pdgLoB(Sl)H4B6iCA#A zQXq?ul56h$)a83UhOS-?cINkJntwi?`cg->pNOLa%Yw-hIY0g%F^*20>rE7&W5^@@!44=zAYelvk;c6kl%`Wnqa9k3^&>y@hl8^O9ItxRFFNKxNlc z*xz@TT6|xE$=e*@?xCS@;&BRHtDc4Z&tr&buNxf{>h&$PB%E@n94c>%s9DiE&>9#9 zdIh6-e(qyPTd|h;d1nsxWX6N-m3*+DQ%J@$L8yOkJpK6A6i+s8CeQ9SqeZF)+^$yP z*y(5JquC#D^sDpKI%^VcND3B$8_Y!;|E`7>VfIMfbArx>WEj6Efs|F4Fn{--pur`h zU}yLl%AATO0ha|r#Onw=8!isg5nX&hucnRcS66HG)GF?hVJtM6E+ub%@58vlRb*y~ zD!nq8LP>N1s+GsX{#Jc>VBHS`VM=7a{2Huz%j4w1A=GeF2JG>RMSWKz{NouM9WCnqhTB^MV@WHblXl&7)!5P*qaI$iR<}7qMg2Bby1iYS86pMNhaD z(rjZxUS>rk?BD8&67Pgr|Gu-x*m4mQRhdD`-CSVCos~4f_b3^0;W!!4eu66xKY_Tp ziNqlJ3jH}-76w-*LA_Qr)fPznsd)yBlf66E-!H>28Z%Jx?Qmc>r_hQQbI85`BYttt zJ`{gl$AnGUPf|lK5cd%oL`_8pR3%)&^s*-&_SXW>3}fEwzyrGKa0n?}VvA(OTT-$n zm?%2rgOkHD-bg)*&gbJrKQNQ1EY&1hSEqsgR(UEC#4SyoCeXM;6*mYW9+lq@(@)t7 z@F%92>_=^!-KvgNGY>;^{xQ*}<%>zL|9lMUltA}!Z@7v5s-$!EexR!k63IKjd)O;M zm&hLKwS{21@tLIl)($*k?Ztm-@#Ia_q%eM~6ioeLO#(&-(HG|@jnvg#4JY2eQi+Ii% z3G$8lV5j_@zBxXRD;ZQ1LK~#mKD$`>7f|KktBD`q%R=m?Y*Y@J!30LQ(Mv4kv1U+v^@nCnJi%z_Z6(G%MG$SR#m?AKkCBe$l=Z_sA!-lWH+kd7~jqi38mz}3J1ra!W<$CC^9tXSn<=D45GVnV%7eC#+O7bRl z(UHFo`!xto zi#NQe z{@k*BOyBsHrYoLATkB}LyCPNSm&2e3HJI2SJcGjulsZL}k_<5e<{NO2&AF(9)Rf+Ne8bU{j<^ zcC-}GS$@q_fA9~Tdb61prhF!AJ_o?#1~+Ckm1WXB63h^i+W_Ub2K zy|XFwyB$EsuidyOY=HQ<>?9V;tEk@gmry^^i=Vtvg6$H!gYFIIx!`e$a8X5t-Sc)h zUnvM4GAeT6{LUP3v}?shmE!EsZf!6=@{&%h?%|y4?m$|N0Ua%61r~QL@Z&E9eqrwv z8rnUH-g2seDe~(<^1VRDV27wrxCl2{HKDkcHeRY354pDzK(VNo+a4Lq1ugna^yd|G zZL<#Y#=S|j=G0+u%JsxK=h}!R5#f^Q4;cEkNLcL9Pv1AHp~v4Ik!7YdG>!Aau3`GH zHA4^`HKsFTS2UvUD;G$59}1n12+Q7 zfRPn#{T2hQ`_IubPj_J89Mm0)!@m*fP^UH@+xr$#2altma59I|w|>-KFc5w1j;8Ih zWALV9d{JI!& zqvRy-kiClAvA^0zB$z`LY!||{$vYv|l+s|uFXY;uI}q9ykIz?&conbTXgYobH8RN+ zEog`#xi>nv`Py@6L(oU^ZF4m>PwOY%{Rv<*S}?W?*zDwlPMlC}j~!{g%#4wZG%l{3 zHm-_<%^Up4#m|Y1?Xfie|A6Z1{{qz&{|&0!ekqJ%4_Ij1W?PK5U6%V0B2GrJrn9cG zCJySh)#+-sjz1Wd-9HPq&M1M)*DpZuy*Ad5>8EL-YOLY=iMIa*`jB*CcGC7`q4 zj1O~kWcv>;WrKY~`TO%O+ZYerC6{F)@Vkv7`{|PeZ?aO@6yj;X_xf()9pld+bV%{N zPW^j+%!SLy77m7|v}~HUWFiT*Gq{_Q9H} zt?xQU3NlniL*@Qt-uze_-nV0(QzJwM((O7g0&o0A`t%@&Qf0@UZj+-@N7}uitCO>+4Ts zUxW@}@9j%x@Aj9$sVgJduUYB*s#+;n!fqkAG%}Enl)%ZI@@&W#86p5@>}(;uw_s?n zuqs5J-&FSiW^5Fa$RB*d9bddqc6$_`XI{Zu1Y7Y=-<=^#AU7_h%;Wphr?c})R`I{G zn)&*sqpWn&a#)v}#Ak|2vBOL*fT%)=RhIaK+H?Z@bwx0==$*r3Lsi*`$*pw7ffaZm z$CRBYD!}r!$6)fbdg!%21XaIRp%ib-mp|DDkFUMM-KSLfdZ#Gh{I}wr=eOY4M?2ah z41rnbF=Q^7HOcwQ+O#Ac)O zKm>^W^xwh0UC1_lDkiP%_Jk{Vo>hFHbL=tht1zwGbgn;x3Wc;LgkSCrC zKMy^I4WpjH`NLB|=a>}V^LYsD-aZTH_xM^(P8W5gJ@6liEy6mM|9INeAj9yW@&D^- zM*n)+|EEy3rYs{mb6uzm<;a1~8jMsd!_9Z4(PG9iQPSyEWM$GZ?v6%12Ro~Y*2ip& z?GC}}U0U!<-WE)R(WLSKPTC^_9lQbWThR(`i6m0;en&3J4x>J^x5nk5*#S)E$sdOE+jLukZq)` zSiJYh{9beaZ+_YTnV~3*Ej*1g@7Nru;&9nJsPsI5jsX*=&EhN82MG%WXFK|jT z!}RnF2D>kl+liv=uAzCr8a9)9GYjm_uOv~b0{(u{B50laj{aNd!N0IXVi@)c;x0Ue zxKx%I+a<(1|4qT*hZpgR@g}A*z=i*KBN`jV7c%yijTmn{3xfSRAkpd<(kagDfDwn! zz^GPbR0W3CRq-!QrxT^_D)937OyXB(V&^-3P^cQsPJZr##+RqkN0n>1ErV7wN0wJd6RKZ@^F0)DO8dC=F~frc4TjK9}iP?^z<>rIb=RrVcM+}^uE6g^W zJWj<^uTmO)k_0L_kv=muUhj<|l&fBZWl2W-7yA>S*0YyRSuDj`Cm)91Q%3yFUKb`b zuO1ajGXSn%q($}!TQ+Cn+Hcdz>y7t_!=*92gP#Gv>+L6|$u=9CCtt%_PsFQpPxDn* z8XzRl4qLaypoyg=H0C{~d$T#Pl{CUzxgsFy`4X#V5xV+{EJH|Pp4|}bMBsoJZ%*K7dh0Dtw|#G#u9- z4`cme=yJDDAiL3;blh`fS8wIXG_>P4qYQs_Lk8v9>D$g)JL7c9MXZ*f5=pfJ=Fsls1+1Voj98%sa-qb-)F78Gzy`3o1qRz^G_(o>RG!V@e4Z_>6W_QeLfq?cgd`0#R z>>8C0LyPuv#jEUabdkMa;dnLJ`TZ92xa4%rc9F^OV6-x?ZYPB`>n=k6(k_~vQBMyv z3FFFQNj^wxh_;GPg)u$xRLpb_xPKqZ^YVej=GQ-*aMXtQv~&@Zj%IS?(P9$w+<`^E z2Q)Z7gBkywVHZnU^C`c*S(hkJ;+L=kZ0I=RSS7}fa_gn%vVRfPjN`=WzASIGLkPM} zeEGZ13AAvmFjsd^48Cu%q_bZuvW|~8;ucPokGgmT4!xTPsvj3K;;|dahjvq%v)q!6 z>5PP(Hy6U*2iM?!rU^*#+MJwD0|xmNkW|rwa6io-B3zW9W0f%5#hRgZNOG;5%KVV!0W#+KMrt8h1QIh> znJv3fL0jtv*zxS*x)WPdX!lVB>zsW4qkS7AGuw*w9NCS7;~nZ|e0>I*3(tUIVK($` zu3^=KV_ESzOF$~+6&pWhJ-=$92((`hApz%mne0azX2~;OL8Dg?{MJ!NsXj?QZucNb ztX;}?a9OC^^$+)DD4BUDuHtR@rEsL^ zXE)ZsOzntOopSkcc#(XYym%MDUS4~Y4jp|%Csc1VJ9u&*taA1S&4+#9t00d{v0qPx0UAx0=HgB^*9lv4)4Zo^X=Pf^z zJ9NdMZoTIKTiNZ&Ic)k&B7!}+_~uujUv!)1)>;F3dyOADTY>pENZl)qNj%+UGQuAG z$kbM4_7!)OCd4?AL-J?vhrz14X6;fkb$EccaawG4?RpI9@T|k_XY$#Z&q}!UcdYnj z{sFjmyn~s4@k=~hU5LvsWZ{>$)%^1_v3$GQV$>CX#b>x}U=3^~@a+$O8tndq`}jhm z&O+%0eya~bt?4SDULwUNiwWUY4_h|d{TO^2EyV7LP^!~Ayoyy+U4pzwCfoeM&1_$6 zI0}^iV#D1{u)*Lh^y^>XN0*GoH9p7LyW3RBxTASkpd|^J>m}H(v*u>kOZL&`I>Jv~ zy%5%&`OLrYttL7)-&nmGbvCPgiCN?gb+cjXU^M%;g)>bY;>SE#&z8*oKxa)wTqsmqgDjEn8|;!GPa7^LQAS333Ut|VHS?Ou?FFT}6Ym(oG}-A_40OS+G|Jif8+ z_WL+=GxR{K^gfgok%yggBY6|MKDzdG489f8$B(1#g9~SiL{m9}tbz%P{28 zHFm;+%kbqr8BsLe|BWz;|3L_m|3Mh3j<|Y!5LXZXK^F1<(>q3iO5uTH|Npo$Zsb?f zZAC_2vu({oM&N0s&=nyehn+)AlGAv)`m!)Bvf<%^aRK{1;wgB@=fIWFPms2D2}~3c z!4ef^T<&MVCvH^5p2L%%==&d15oO2Tp5H(mFFnMshmPXdX@y*`Xen;*E5)AQsbt8j zye66}fQ^B-m`3ld5aHXYCFEVTGzA)lB z4`JLEU$Rx^By%m(o=n$w2IKvu*dHBF{~Ar9`6m`rAL$=-T8#?yowLOC5gg&;pMpA( z#T)tAwR*G+xWrayh}+W$ zh8lKoS@k`B88sHLKaN_8M>3tP7_1+72s4*^(@m!}z_p_tTKatO$Wbp?t02W*%ZtXf z7G(@}UId}I4o)>go*I2i1^MMFBc-tz%eI-bjzLR+=`6u*UUk&B>o^JQ{fo=v^+EUM zF`Skw&AVJFB$r#TXBq^~ za04U#MC#Qu4`%*zW{wF)Btdiv#GKg;KESjy zr6Eb!f3cpPiYs6yHb-LV$FuNRR2pWxCc*Z*PNusZSgL+BgDRY9;ns+4L+u-v=oLvd zc6qDP%Cb zBJRkFs;y6dNOuZL<6x%*@BhpfwWWqh_33QdyG)pfMNUEUD>LESp}%zd#?xfl(OY!? zd=oM?^)&tz^izRi0(^B4LC?AEbndz+&hgeIau&v+wRIhNl_3FB-^!sya}#8AJ}1+) zPLPVqBbZdB#Oej6LsZ*({MwO^&c9c{+MZqTedjWIXM!)>KPZfqi&Hpq))1r)SW}zC zV91*p0QcUeVp(_+{A5*$pg0D1xPRtG?wb6GF(eN^{3V(`X5exE9IcYp6g=trP2F%j z%iY?8dv8U8q>l-P@1Dq?=SRby_kQ$i@Npa+tc!Q9G%-&X+2B~`HoCv1h~8=3PqG?+ zlJ%0gxP5sXlP!~vQp!1`Z=f7LNlJ1e^GxC4@@Slo0||Im8Ra6dGZV{v}vpatPCYy#*ODIkXp8pzw?adUK~Dl*Px9B@I0O*pNg5B~oZH zZpN9@63FJ*%k)E-HWY4Qxf6#kGNO-N=;9I!I5Mx0%=(rBUY4RE8ipFNRQ#dWN+K6E!GxnPA#>4QyjysQ5uNZEtyVZw=bH}X-;Ym3anEtG zZf237rC%6y-b52ODGkBZ%Nlgmn?(5Ls?PXG+d;YIJUDyv4ospcxIY>3dafz z_-Qlq$#gbN{pStV`7X3>8bHWkD=n~VC9_h5S^xT#q{4O`v>!2slb53K@%14{Ff)Y2 z2V*chFp17I@1(OIUt;VvyV4G+CgyOs zKm^a+o5tm?9OCjOm(!c;!qG6Xor+Xu)^>`YA+2}EA#=f)*q=B7V$&;O?J_m^na<$R z+z7Ns!#TR})`1DdX5X8u+6% zmbi^I1ucF#=C7a2HRaqU3GJKUo!}DQ<)zqxfdc5pC>l6zCmC{6fys-CiPwZMvR)w% z2fH1?Lj4jI92%e(uC4&b*$s^F2o{p-#kl{CEa^C&i$DFh;H|#7Si35n{{6NY!bS>t z>1%&XSiTw}%!GKq#B9{xqzP$3$LaYS#-MUxH*{pS(z|*$XsP-+{OhfQt(sbJ>NIKRv_bSRM1y{294hL0>`zBpu5r$@^(ztxS|@&XUm{P1A&jL6&Poy?bP$OE-3bo!)Fa!nG5!JVcb0*azVxi zc4 zn^kp(-pzKvrYs|nn<9rol|po#WfK*j70cP}GNTg~*yH>Ow&3us0_KGkVx>eBst)CH z1}k#8151ul_FN(D#9`u)egvRPf?rd74Cy)prAM>8|12 z@AF{hBun@iG5Gr8ch1-2CU_MmV_1GJ&UNUd>tsCH750^MPIVSEM+o89j7ZcMjRBE3 zLvp=QgBAxb1f|QmSl%EF5^@(zQmZa9j{cooNKZJt7SrOsohZTT9Glv2K2q#+A-`J2 zrwx5)ekaY28sL?&9@1i@aPy;Aos!ZaQA&H7(r?v`_Ri{e96JLwW>Y>DR`%{*I?KNfcKr zpP*i$Q}Mtp7f_v-3lo>Kf*^JpE=n>54R#xzU;UDPnAt6Pq12*#02_AJkaj^a?2mT^`1qG1KIq+3yDpm(3*BtOn_jEUuj3 z4{^_fA$ns8of)M{e(VT>Lhv_Q{kCpJdtq5FxwH}`YwbPsC(*#n5@8IL&cTBJ2ES#rrM4f(n z;-1m{C^_pVJ&?8o-qnYqvH5poR!@cB+eh}>nl>6f=>_ZzE+n6&-5CcFA-ch@lXM%n z!LPh*{97!5xpuMWa(N9d{GN&qk|}7XsEvPBEQqS?c=Djr4t?1w($?XQOWiKQZINh_ zd}t>1$f#xZglm9SSs88LDMBi`7^qG9L@@3QZrC>t_o++)DIIyVm^L2AOx7mxYSCn% z=OBsemg18)x02rZU35S*0@}kqq4SnRG#H*r4Ghdlw8nl&-f)`OYRyEq!M&s%43Ta!xT`poI1iQK!TKXe*Kff!=1cKZdK65Q9pR5v zt01SR2U~A6U^IUP&jb{p0m?w)@^tb_!HU^rHibQP;1VS->KT{DW5h31KouP`1lAkI z!8g92gxQzkl%Ys)JKfLJJt$zdrH(?<7=xZ~dun-w(XiO1n@(%dLwD1sWb4icRIZ9r zx#%Wp+q?)=9Kx}{wuKr!62PU_lU(-9a%`a8xbS5s%n^^mRd=g!cytfF(_9Oel%(+E z>S|KGjX_sr37;H$r_zO#|2x#Z4+-aY;eR)8;-!GBY(Jt8J=+Lx(`Sn`UIhg*RlQzNB)kR1RE}z z)50JLMjxzkwOBLf_OZ^iyBwM5lV4$lxDMKlKa0<=@f43<#Nm+Hcxqkz2AP} zuA(*MappampZAw;IyHyA{6ZV|wywd63&%rKegnDiq?U|wwBrQR6?yBT6QIAi829?B z@tx!pJ3($fs=N9itt^NB0V`b4`v&rF#f{Vl0C;R(>UvKzyF#?T2haqM~Prds^mOq?r~p{>XOTDn`A)MjZuuEq>zMJ1CD zwukXsgg^Q5;WSB-&V#G}N}!@bh`l3KLG@bI7=JB$ewDL06dxC5ODbKlXD7wC3YVyT zjy;{zq=bjp-zA^4w}APaC{jWznMFl(pfiTM zB+>`HS0g#&cMs9nNt7Bh_ld;jnV5Pj70jzE@!v=b*(v4{jn;b2D7l}7J@cZVRJ{Pt zF4H2K z&Ug;v=10-Z@^k4xls{Z?2_+4V6ZosC%OLkgG$S`=6tJ?1@OR34Cg@H)yy%aGhzskX ztkeR8szx}^=t`u`De!KcCfX0?kW03a)KhB%B<&ZY{eN$u8r4OU?csFNNc|femSE32 z>C;A6aX8j)i1WKtY9DRP!rcx-wJu*0YtKK92CF@b0FAS$Q{f3%IHFZ+#_Yj=qX+54 zKNF#4$c*kODaN4}UvRzm9s0xI6YMPaz~!O!WL}dl8C#!9zlk}Zx?dKS1>~cK-YAs+ zD$mzSw?kmwB)sK)hbtN3r=lI=ta)fYym1`GHkI4~$47y%0HdjdLmW=~a2M1&qeu@b zlAXItWhDtpT@1>@pK43;+F{3qILN%<${ol9$n0w+qHHHAoVl79+sLzD3N&%} z#v#0M*$oEmgKLYvi_uFjmNT!Wq*B~&3U|69L3^SvRnr>*aL zRN8SSa>Z;+Kk=JxnR0|Q8n1vEdx}x!+evIf9yCWW)OB|{obdTbYzsGG)BOq(#_qwU zamMg-dl4=)=mM$UUD(b%C+oJk(mluO;56+guCAI~(74%fXSF;ZP`a49oB8tj7tF}9 zpTBVQrePUL;2!=*tB}YN;Oml;lCT`U#ns~Q~Zf|mY<=c`W@g+ zd_HF)=K{_7lUU<;6Z|>vEK0PFL7U#;TGDKQroO9*qni>+RPLoWHaLO7a(&p6VosMF z_(?juH=%0bRJc)mkjAUo!f`!e_H}iEAWO4_+#V$Y#oHaok$>@wQoJLM|11VkmvvA} z^a4KooJpI*XHh|fDvh=tv0qme!8V;x96LUgnJ};sl<6%dp+tdQG315cquvURw8;Rc zo(vm58)3)x%@{kV4`P2dVR(}a-bo!tg=2N7e{d{OSpEn~qgag2y^N<56v=hJ)7a1} zPin2C@$4>7)QznoPn;9*-vm*x`soJq(xPC;mZvc1@ORku*BhsNcz}z}f8>U=Zqlcz z2Z+kWXk74JnD2FsV%8*?lgD6y`mz)0=aK#5cm6%Ic2y@BzI2Dyk^@xS@ig~Q+l)U^ zyAX#y&ILwQ8D0FYaMmqGD1IY^rn*c($!c*J(AzU+qmDTk?Hk3T6$T}GyP z_~N7XFHECk6olWsBS`ccM`!F_38%IN!4BU!IOe1pq{&8bAN2N7%fUu&SL`FodMsvs zEXXBV-yGqvtRwDNUC2FD@WuyY9bxs_>1eFw2l{>o!Eas+M#;BebCMFRbxXmkZ!h8u zCv$35FNY__HjvlK=gIm*ifGg)17BCDfr#%gc6IM%Vnk1ZjD!Vh8C`>zib&e79**D7 zKVkm*1cC9FU@$tHfbyFYN9sxd1|P5mp=a~KYSv`@S}R9rOEpSJ=)rGJ8;^Z28^N5n zR66n-*6Qv@2OWEIZ&@!rGIb+%CR)J-m36@Fv$8lBmgh3~by$gRzaID6V* zsxL7Yj6Y1p%%P32tD=c|H9f|s>;p*co&@r4(UVO?&I|InoC4>I1aPt>cV>I8~D8ECOpuRhOJU|*gi>` zC^uim4k(1c%8BroGW?F8=V9j$Jv`bNkE%gdAZr#wMwM-WQI)e`dc{-Q_FSyc@y&B{r1vo{!}^fL~=2B_%lPTCsRW0Lk? zQff2<3lAKI*Gs+W3{hoJz3WX2s0H(2`E=BcnL_R4r%dq$7M?5NH~l}wM3Ja_)NO5OaLbGlJrzE!dxbr9AA}8A2iNoPMvMR8(C+Vg%8zH z(*6wX?9IS8`a;};FRM^=y#=#h_XGx76jJ%57zj@3hL+Q*;G1QQYLYK#y!dG}n=1)r zEq_VP_z>V~CUJd#SK)m1J6!!9OB{VV9scSGxUiXRq##X-k9o5X5*_xDPno&+Z@w;G zP?|z3s-?K0Stsa*3Re>FVIQ{ZsDfF;23We|94txa$dP9q+)ME>kWxDxqE6?bikd5Q zg_pv^$V5~Zu7HP$SBR-t1-&C;Ow_F8*!2t2K)>q~wkuAe?x|+{p|pCOH1Q;!%~nFo zy(eg;@I<_o6^YAQg`hxcJe%U~4l=qLn0j6m=WS@ltrEpd*BMFZNqP?3#|y#h{50DA z^&Z{2PMe>6?=UthEXSO46&SoDnp*i*z(IFeylH~aHti@@-%rPWO?4dWc?133;^7>x zhmqEciNy*^r$4Y1JSy?R#P{n#G1wiRx4frCYmdXb=O=;tcM}4Al51}*G=g1xI??I# z#E+rN;hpT`nzt`=NXO+v^!LelB6l*HyCQgvzuJY_EYk|Kh>k~t*j=a;?u+nfQLX87 zIZT%MLK>B%i0u8#Ouo@ZxWxz1yN1H(q4tG9j!9R zhx4x%bCCr>IREEAC~i$a)#x>taYGaQ-sVEzo;~OqkpO>GGU$DrkHq*geg7&Ejn1Uf zFG{b`Df}FHG^LXiS(jj$a6GYZYC;>?Wb$H9I8M+Ng?+I$xOs&w8IbEI=^I_y36B#Q z$BE@sJK-hq8Xkq3PtFmOxwaU(D2e-G>O!{+O@jlzQKapJ0W|ASu>G(T%a{UOmLb9P z{vCkAXS#TZ|3zd6?g~c9I&kryDO2X#!t?Rko8OdpPol5;dr@kU0@O`82mvlZba2E|iAY>QC6|wa z(KRtdNq#St)cDC<`}UTW*A-*X8cVc&I|82;|U8c%A9!k~HYWz>m}rcZU_a8S)0edD!(uLkg+7tL(9&7fsx zyrBPIJkfWmC(NepG+^^OoIYaEZ^b+23iF6; zfhE-anlSu0tPbu|7I2wDp-lY5VzSZwE;UYWCc2-qn14Uw!F+d(L&5GY-g_XXJFpSUGRLe1v!*?j!}C*!rNEvW5#aVhnk}9jK$t`Y*^eN&|Z3x z%TU-3pQa~a!x>?`J5daUHcuzfHlg(5;$Fy?{Y!6|Hw(_JNZ|IXFTg9F?wFZZPVH2q zVdbw(2+^EPN+%ycixVqI%AM3V<4X~1e9Mj@fmSafj_Sw!XNfZsuk&RPyGQ?;P{M?3BHIYQU0+)1_V z6PZ&g9eBD=k;_2@arQHKWOO1r?ax90SvizvhG#jlexm zsn)Ri1J$|XPMb2!@$^U^JZVoQE}h&6Ugv+nQG*P6X_YCeEzN{$5>3Qe^CVj8n$h)j zp16KN6*MX|kqEs2ddp6fC0EYTOq&X3g;)czn4b*mCYK7DCrV+`a0AL-K8}@=^_Cg&!BR~b9G*Ce{Lr6H`+uLJ_^gHa zapy>fqY=z;b?0ifRnv`AZjuVA`(XU$8r^8BiUk!fXtKvxTHdt^ZSUvN9gmRv)%1#Z zb-o~d3DTf5a$mBt9l? zz8A%pXkbruCA0m}4LYmv9;Qer;$G!tU|IMG(tLcOpe37%*B7EEN;1CJo$$!DIT+>^ z43S9(aph$nF2^pA*n4jw-m8;9<%Brjbwic-y%a%79ZS&bHiBz=)yda!yO>|b6;xZ) zXCxZePVGb^aZ;W#ez;n~xm*oHqX*wc;&0l}?jjG4Ld$8h<6aDEGohVk0LRL1;qlxK zGfNdd}gtUAD0O#|hHl8%|%oSEAyheDO%37WNK#uve~J!cCVi z5V^U@Sn4z4S#6wzvsQO<;%Xc*{xbnK=eLv9hw|tN5jS}M{*R#Ad;)RhMM;x;H646p zNDSSsqQ1pm2>ZGL4yez>&6er(=?;DV(BM`)R9FBW8NM`YYc6@3-$p)S8qM;#4Xy!& z^i5YZ-dgBJrk@POhMgZEFzzPFh*(9R#U8@AwdJrtF9T$+oQ4nKQ($PKH2aTCh6!qp z{OdKM)ZNnv$JrnXv?ST3aw=qP<6-dp)WEG#tEXju>M*$80n)NYG6TyUaiWHW{d49cjjc|?2l5&?x4eZb2)j+jthzz}hH>ck(gmeFBe{beHn6$c z2@<`x(EVX&$W~@7?1_$tL-IoOES|?|v+=k!?-BiwRRDcYnQk>TK{7 zJZ;?YkZmGarY%pqHl8Ak&~BW4EFDA79l^NchD?jwMS9wPGJd>T0`{?a0=0wsjDPJ# zh*&xezLxudam!U&I9x4QzTym+Db!MBi)32Vvlh!X`p}cgq7Zud8TaJ00t)*@W4q@t zb!Rq#+3Sp9Pgas@v8m+Tzs1n_YC8yeMe%800+uzXqQ>)caP?+6_(L>JZ7gC> z-HL^bOZhnMfj@EcEg(0X4ndXHG4M2=i1#W~Ml#|?WYNPIG@l$tuJm4lwf?o(X0ig~ zH$0>^z5b+W(n6{+ei<}|7QmCU2Wj&;85HBMf`rm(uflu>Eo`gzrzLiyAHP{MDKGHeQOJ z(2FETO%JfD!P8+(>Hy4j$b^fbG7$doAgn*xP4WxX$=Zrsn$~)N^Hx4ZWyf2Sr%A5p zaLf?4pDLuAA_loRtNx)(Spz=vHAMRyUtGTV3Z^BMGQu0HV9~X^ribSyqN%wh*NBz$ zX=xW7`|1P@=(q>X{e!eK`2fTlXA|?mEc)n(1D4)R2iJ&gXi%02X8WX}u<$A}!5JvC z>@X0kk<7)te%f<47Ta=0`VWTZ(Bjd4G@O)#d zbKrQB43r%#BVxPE@shs-v=wK9@!S1mQp8>IGwT@|jJuBJQn8@oVTCU{>gk?TbzEAT z2O>^NT)X{!!S63D8F7k;M}{9N3g+_T4kfTjX4qC&~p$~Ya-VOG@nMVz$ z^wf%O z!IWbfbp7CGjHvU4?!A^o@k1`j_RmJ4Ee?W~{Ak=&{hJ2HeIliKi|H!lXd!VDZ1fA@Qz)SG~DNwh; z^Y=_KIsY(O-KvklUntzOd;(w6XW--alCaM8`Uq#vg4y-C(0Zv2iCi|E@KD7IX8U0H znJAsr+lPPZ%IUxwFU)v2hjD(oAD%?=^o@`_Ef_t!)@P{|9$(jlk*TxrY5jS`GwYG> zsv?iapWs?*3h9&8IiMw{ELde*g_4b?FlnL`SfB2${WbFiNG&@J@rM_4BC5GHXc&GR(`26`y z`wq82>;{jKQ0Wl}cNgFlv3$a%4badJWmq{ko!$_q5MuojjV~F)xX~?mvOt`!oD)m- z-$=&ByF*0oNj>gd(1D$hT%Uc*+K!?&0z85pSIBoevizZ__kh4YGzWpv|hNk)9pmi?!QevU z9#WT^p}0&n3B_i~u><89sG=tDiGN7WsRLiVEWzJK61djsiM zE5|lJTL2Z??=Xr(JXvw6fRjAy2KKjAacay13=rA|cb+7G`}In0Qz451qq?{*<9MiE z$HU~-%e8m5j)tGMTT#}{i}x({MP>UVAi@;`b10vDu3iFhW9=c~UU$u7TyNtDR;@pXf#DK7z+WLsx;fCi zI)`?dr$BUW4rkHkh;L?p0rkn6_=7g$iD$YnFQ%DUnHP>fG}54GE`V8w0Xh0`7D?~B zfCA}fG(l|-?$nXt%QSv7>A`0)cc7NWxURzC;d4~s-x^xJSAltDa8S^pt_ukzLG;U{ zA_(mtBvQd2QNndUaqU`7y+(ReBGM!LN+KBak1VCn?q4Bu5+~z|f-L&|UOIJIb&nHq zw?v<{+o@CH4{F<+1l_48F{LLOu8d;geC2s~y(gMnE*teVU3D?PkH%oEr${nQo z+9m8&PNVHwWl%UzhH6Z4qER|3*cQ12(|)$lv*rzG=X;wNEqqRLRh&TDB^QmnBk{rv zM>^n^OOv`zg56SOh}t{`Wt3U$-94FoB-TWo7cameovGx;mOJF1xfy86E04Id6;e++xX(#53LyhQdG7@V+gKX3ToL^Z9eF`?2}8S|4uF z;La9e^CE#t>{noCeZ58;J@nX~l`QmL=)kLwa`DZ25&q-jEo7d_a!`{!M}vj^Y1|zr z+TTuK(NPc5vSR_&U3d_i42#fag)~+@zE2Z#UC7FvA*AP?42{^m1@%8&CzltjpnK>R zyizqEeY2L3O>7t|I?{Xm`C}4lk2^~?6^_AanRDq2s}2+^8qFU(JBp;KZi2D3!?h*a zZsgBpC-{A-4=3k+qpP(;MjjoUjpx%AGKb3*@QJKE|7E@$JOBC=%I{Lg!1Z&PA>r-h z{`{*r+CUo7C-3+Rp3ESlSwQ6+a)g#Ud&85OOPp~LG8c5kgG9Wo<1>(t-) z^yVQrqdXDb4>wTLWlKOLWf~js=~QiIq94NTf4JBPW=AwvqVco+jV96u3286dyu6MnCnz z$@|B{)M=Z!xKFEaY%e?s9>f1Ry!mbFkhyKn;W6 z;fA}#Oz<&Z1_YYCPN@pGDt)0gY!0oN^Mac^A)6f7SxZw_uSEqF0bKhR09_v20Uxf# z*-K}TW-}*Tc;Yts+LD9Yb03mWwEe%vFjXTs3no<8jPun5n;aw58?Wl<~qF-^KP7InnD z$SIWsa*V#D&eMHH_U=PsqxF)o%2|+GrGS%`jK?RYi=g=KZqjkx61RyRr&oUmfV7$k z=~vf=_a_9v>L!uF2SD!55JhEm8}N(Gprdz<_(Gq<>9|z^AoS@Jqg!mq70iA}JY@+U zT4h1*Xljuy#rsK>4M*)dBXNy@@P+FF@s7L^8Te7mxHMJKQ)8!)u{Se7N^&{;r}xMo z2oktFZKAz4?KI_e1-*T?k8TO7pf)}^)I6R=@ohS&Fw=(~I<5wl6Hk!iZn;qUd?KFw znS*Ao5Ac@IGeIYlNvcb-h?Uh^@~qQ5&X< z4D8U=hcEnJazI9zZnNuWKHXNqyQ8j=(91t|<5gmWGA{0gkT%&e1 zEXm2YBSy1>VeCjI``a~3oCl5=QFwv4uf0#B*2Kefp#)4DX9Cf&p5(;iqs-oil~^Ko z3g*Uk!dpTM)}hX@dE7^8oH$#Olr4@!rx@+SmHT z)a7a^xge#^m~XpDMDJ$6*HQCW+m*+u@TYXrabE?selFvEHvPqil_f;_>lkjpvw-~a zkwBVROP1ZqAd@YcOy=kwBC$7);rzN0|3y9lJuO@4k0l&@xH%b(+SK^UbGOjS$9*Jw zT~BXX>;!kIowZv;&f|uF3hHx5oG$X53F5n-(*CBK_{f7Nof*fVT;)maibeT2PW%t` zvKuBc%P&*4l@i$Z^b$xt)kJ1wt=zz~RQ2;~x_5y$6kS$@*G8Azjh_?hxZp}>n0P%3Kg*0@M{hR7M`+>;l`0ad_YA*0P9!^w57XV| z%iy`X258qY`fBtbaj2I8CI+t=rz7gB<@H_qb@owuE%{eXWYbByqwouToSDpYEeOI-vR)vuAOr%+Qy`@? z2#=Wb(J@oi7?J;np)-xAs*B>VA!JI1kPwm-QK@jxURM-NQfWXWX;4a~l8Q2B%q&Ah zGN+VOa?f6uGWDuZQKW$eC6$tj#`k>p;qtro-fOMr|Ez!^CiQA6xNWV)`8p9~_kK@U zyH!Y5Db1h>hq|gK#>8XcyKu7kBljlSCJrs@W-(Tkm!N;46#v$pe0&f&03Lb2>Bj8@ zRsFm0(3VmxUlK!m{9e%2mfo;trYok6)UXQvK6QDGrxpDxAGit?dWH6XTKp@Vng`CA_E^c-9jIfr9~6nK(E3PFMbLq z-{0+`TYbFgqhnvmJraly)~%uUT`%F0hBdT2C6#>i+5j?t6-mGTcdC)*4&ocG(7F4? za7C9pdZszyucN_m@v$GX>s%~t%^rX|W9IW|#Y%jBCK&O}3cPi~4W3>*O-)Dih~Rc2 zOv~uT8y{TZ+M*8h=LONg|0EE)63LXvt28-I3G;hwscCxvexH=gCKTAi;Tv~hlGjR{ zB+G(rrybKvfStfc9}s=p)R)=w%B@m?)1d{a1h$2e#tf zZ%d(VnG~s97>Wm)78A{LDWv7$0XlT?6bh`m*uxLwcoWRhK@!x!jddrNRYU1iiw>%p zwF_?Zq;dM~O~Tx&VKPfm$}j7 zORnuLB5y;3>Cp$Hbn3?(I=ogL6{m`Vj$JB74UQ8!rzR4aU#HQbrx8m|JmMXi=Z)24 zjUgjmiGNk@H2LXXi+Q0wcs^|r`6xS;T1iBLdj*0?69IRP9vLa-JMS^UazKImz3~FkvoJ`zs9-VH?X!n zh8S*YLu+{xP&a%+1Y4Iv=$0`0&b^R%vY?*$OjRaN>i5&I;46Z$Tc(hU%FC!`Q4{<1 z!WvRJD}h<`^C*h%GhXX7u`XseMHZ*~8T&>%`W9 z-cLtf!CGnf^SYP*A{FG^;}&9hR|FsKQ=wb04AH9Cb|z@jdz$BGg$^!>s3Et7xIK`C zElL46*QQU9(VdhGxb_!>9@ADy3{6^qTNG+|F(}I|^MrwXE4IPUwp|zL@w#%jQhEDCK z)jwCENuxHpzjx$u<72RiO2Sh%(U93D3Z0FUU}lXLhIc!`;DIkNrZooB6z-AQM_sse zXcqcOZUw&(Yw9cCO4J(#aOBcTdiKgA+#HaGZTs!+vxcdR&~kK$pPBrmp(+?7EaCBPX^pGi1X817WenwUHtv>g8k|Jr$gY_`6Sr?vZ zU0@UAvY<7YrH4A+(xFG5=-d=ZP(+s4@ZON#aeU0X?#=rk5sfBV`_SUTY;xkdHCPlT z;uJS;Dd=Ag{Z>E(Q74Ros8ABp;sQv3{_*H%u`fRU&Vddp?=NonKQ=0(d zz1ld&DV59GsNnV$MMN*+D13h7Mck5SV``Es28J}lnsRSqb)b;`^ivb=o2E+&y*!}o zVgee5zacw#SBaEXA>PjYO+FFV9~+T8-~l zH4p}ufZH)$+A`M`ua}$S+ZAt7^KLrM6ms>g);O3PZHO|vRN(d>b5`wUGGoEL;kS>T z2czf1Xs!{0PgRWYSC&1E4NRpyK9Vr7HUo6sJ^5$+ec)yE0yL5xBr6ZBMT<}={_~oJ zAQBnIUlVtO%-w$;A}${kuFKQn&0cm`Se%`YyT`?1@YNmUrS)QffOT}@i(I@*TtND@ zi_ls!7KQeMG?=T24DDx-Y7JAiCvi9Jx?D?2o7|zVw+f3EG~g_mFg)Dp0BN1}__ap_ zcZ9~m2a7rIY2yLNIhsv>+D#KSSjwOZ#|Jd7vcs~=KKOj_1kBtKh-7m<==e5a&_!9H z?=vg7C4Cz{ZW4i+(>rONWg7Bj%h^5D8s&O#!1>gl%yGv!#_8%;s&MEYEnj>UJ-9o6 z=Qk(Js>^}c4?1{x&2Rd~WP}#1T7sK$F0!rPEZ9b=C%n?2*?6enAn+8v!1wxW+-8T&YgfWvG!mwl8* zl=)NRxj9f?@Rc591_f%nG+|jFS3epUqjO&rhM44XGmTI@{sHOhZHCbIS(mNz55Wm{ zwD9~MBjIhkO0rE*%VtjAjIkO_7@0nWEHKHYH_CK~gLx$_U4Ifj^G4aUIV!X} zNfxEL^=|BFS&d71$pngG$iQBJ!}CUo?0pUNoHG_wmTS|bQyfPlZcT;lyQz_K1hJLu z1*Zp-`0C+HDfRsa(hi@gZi^;N^?S;8eaOdM&9>m>$7RN*c!JcmLKFusp!#VKsb6G{ z4!2G5ykZCKvG#|#w)5aJb)nbPCBWqDWMG1m1&K|I>E^E0RH2fkvQ<@7V7>Oqzke<& zNKX^aZcxD*cP`sf_ZEZyyuv$Iq(Gr@nCqe1!qdofaP8(}!Kfc!O?`-V_EpogX-aUY zJrT1n+#qV&t<-yF@q_h5zP-hKvXBmtGq=~H2j78YZZm})FBZ~c5m!KF`Y5?Rt%i#H z=HYtjt(g8Mjz~IlHCFBgqQ8QBmurLwG-L%NamFtE{wb0U9yVigj;2z>ry8(H+6ZnP zsDa$kXlQo!1D&WH?8W;D^jX(460iG-bXtq?O%r56q2WBWicY}df3djJsECma?1v{? zmHE-R_sQ0(Bn-`qAaPeC;k?8>qId0Gd!K!5#4XdI^^y@`nvcHbeyO%+=efH9gl?kxG{55%N9L6zl-|2@1 zCip`11AESAF$C(AvG4!uB)@h|L}8m4C|hkLswRJkt^Hwi)>~4Ke;>qL`NID-OU9yUGst{TulO2N3t#Fcf;N}(a>&kg7BZ1 z@HE$!P!49%-u?^syd z=Y!f|+PrN;ihSFa)1;*~0e_x4gnC&#>a?pKHXSlzex^w=2B~K7f#Xs7f2-34Gxy>o zWmy{G5r&rMtjW~Vee{Ws6da$gi7R{>@xqBO)TC_#lk5AOyq;S~^5nTb>UtTz&#_ft z)iX$M@#JCLk&EO`%QpC%@tp)qFQa!n=7H1eW@gqVKB^i>(MF|KDz!a}o&R4i*%H;o zj(*GHm02~ht+sRNw$eYCeQzf=mp-P`K1M)vYzO1L*@U2w3l8QQ}b+5s1tm!6GSeKj1m*f!9DG4b^F z1zkpi%UkfRACvP_<`cWCg=E~Uu^{$G68|0UA**)t$=5N((4zkVCBGLia#jmztYkHn z&2nWuHqQr=>w)4wTxrVdB<|i%XJl<=BmNf(&PGSkCa(l<@_FR*%K<@`!7{R;Y&^Z+ zv=Q?ZENFk4FG+0`#m-C_X8*)o$oLfkXYEguz}0scpD-RBZu9`v zGIxf}#D4LIbooijd%ICUWk0W@@;|lkL2D*Bxxc}&b_+0f$7ZZ4bYwi&OaO~TPM}_q zO@gwIvGc{w^Q4|0#&c;)KsH@KJ;t1-^LH)>>o9B7>sn4qa07Tg4&=`Lcsy7u3i6Es ztXGjAxn`*i^l}LP;fWEcPkdUZI*kNQUd<-2iG_t#7ijk3Y?9S>f*G898eNp%ll1iU zjDE;{EPldZkc5cvq1I)ZcV+?kIjlr8g0f(sZ6ZuUIig;fK$dTp;YQ%c=@%PIDpKK0 zyww_sA?yZ?dCtV`UlRk@lA+`vm#1*MLUfANsF$oS#s)Spt%?SuH_<{^yKp)%zed<8 zZ?#b7Zz(aznTt7dr6Eu)g;-R4BKp_U=)8Y7NQBW8);`67xz?r$uY6bHcF#HBukxKV z&Rl~fol)fFyA1lTG9A|!&I6TYGnv<6Q^-}Rd8DJY1)B~mzySYt_;p#0P2V;|CWJ}h zIWrk9k8Xw$u6@kQ84vLRSF_nD|Dj)J^ikonRpfG2IAqQyC=p#n9c-0Y_R?q8b4(oF zHoXbfDT<)}B|f`C?in4434j`{VU~Zi9FA^ZhxcOSAZa#|)V4G{Jn0X6WQ!(DKjFoD zclHA2)ehk7{!2JqvXt&iKL!4Ui@?4avY5nE6MwoK{9# zCO5+LlNs=8=mWiQNEL=ozGr?alz?#EAdTvx{F`};Bc`w^)M_tad_sZHS}6Hkd#{qbo!cTeCTS6ZCh8P+Kb;b zp;Q*d)84_49DrVvk3?E5P;hURG(0&s4p+w;ptbir_)#?nyVlHtOCe=gxk^Q-TY*wm7tc9b9v0-v!9l%C;4LZvYd;3z$$Q_yc84^&B^aQ~@pPP>^Bna=3fPs) zW9U?`dOAMao$ zPgCg^PMm}3F3Ak+U<>@0;ba9z_)(Te$8)nO#q{T7U}+zlSdNz* zQ|ZiG4W#T|12l{r$7}XC=_iFgz^``+Sw9~Oe;Y#Qm<<@cQxo?|$qHAT$wS4u$*?-t z506}OH6P{p=yf4q$cEE${8TSbDDGZ@TR8qsWY;}*|5KKCZPFm5Ke|OL_QqrF1S5Rg zy@uFJDd2&ug;*r+gR6I#;}hXt_$D_47a88fNe*X7;GG~e)>wpkiq1Ihj0e`Z?S;|W zN@{C-mhzLU=qSyx!oaT+gwYGW6_lz|mfAJfN~m+9^H z?NGgZ6Inl+hGo}E7(3R5<6(_p+D{Gg*zY7`q-B8b4%yHTyf&;bT_&6zQH`6+!pQ90 za5|Jbi$7+~UHoC6g+5vt z2Nnv-tKsJSt;~h_GQwvZAFTAtgTLLy1iPLmL0NAwI`dn>+{}wMN!3#Efi`CIv_?G8 zW&wU_dbt0IE>1SH=C?3**xcMf64$UO?Hta%iEZeBRycL|9g8L>jtX;l*ZQ+_4H{LdQL+j0wE&fuV#Uar5IT*ze4_{&CTouwCj zmypi+Z<%PHIHHy>F5G-y0TQi)>81ByNa8jruuZ#+-alp0L|O`OoZbfyk~>IA`dMt6 zuSFAcLvSgHCqusaSTqud)2{u66JxGW7m)~D|7n8d=FYFI@;X_1Ol~#4o_r2c=_kvB zFQ>p4-vILIB*&uf`opva-o{5xGPtcsfsDD+i68rtiJAUbc*stJ1#>*$htCD*HLZ(1iu}%5aAh(x^COZ!o4Lh#rQRObTApdAE=@+ z$qG1k*n#5%M~I|#6MWvj2);)wht&ESbmRI9w%ao?p<^@byCuzv>wl#Wi#O0U&7EXN zxgxmv$3cT%7F{a87{95>Mo9exn-4g(;eHj+{5?O#0Mjp4QMrB+X)Y^JgWF%3XvB6Wz$(n1$%0yPON! zaC7&C^FXbx0GsJHt0&S+yaU9dEetG$5RK&$j~Jbpf5= z)It_Fq(a8uU>c>8M!#(;WjZF0(w*j7_)hLT^FuKl2IWFQCe{Y~o3rpxuRTUg)y3?t zU)Vg%{eJ}VQEUGOd{v6(kY*T7seE;$N6(X+e{$&d{TOT4|l^D>}2i z9l_xqZK<73XLjh~e>QRGCX@yv#r;fg%_I_XF^9sL9dy}C8Pr)=!((F^RxAW-9eeDs5`j}OozSnc7#w@gP-CxBR$|>8>{w@nW0;fV@3$Hv zAL{}#qN+slZ!(#psU~!n^<_h%`|0mu1^&!$<6t~8eIetM4t z{o?BOXGgJqb~A(?alq#a_Lx00uF5TZGttW!%b2G-nPelP)jzBGNE%Xuq$#=uS6 zBwA*3g>-yiX}Zc2azTD9PUC!fvg@O0v~(St{)pf`3up4(I2Gz{D?zVY0qvy(s#M&; zxG|CZUMr*zK6&B}Us>F9C=}<4I1&4MrIea~0@*<$?D8|mvAL~8ZI2^$mwrc{-QI%+ zJ39oS>0O}KkdI||m4!x6B}n8zD25u^(b81z^S{1M*qc5uV`(R;60F38=yvj_w*(qW zCHX|W4aG*B`8rX}WY6(8=(Y73;nhdj3;_-C$Cif zNKn>a+&rZc%#0K8zCkWLvTbKKIUK~iZD#PzY#Kg#oPraY$Du+pc<6o|3pP;FGRvFs>0Ch0N>AWl?Vm*k-7Sz7J))VJg+z9xHu{QQ6j(j@OL8sx znYBT>;C5pUU3uy+nYBEcuj>AdT&>(ombcol4mG+E=6eM4t`u_~f_x(QU_*TS85+a6 z(d-uova6hwQT_f5xY=+M2Bik*%e{rDS9csAoSJ}T!%Fz@o4jz`-Vr+Q^Hlx|)qdz` z9*Y*PH^I8thz8(Wl2TufhT1L=zxx5n_REFePxT=)T?Ho(4O4~t<1pUUfoNMVg#hU$ z#(cU3UbcG{>WES8Y%?b-a3|ITm>5f5{PinzX=GDrH$ z821_-u0OmKYd>+^S!oU{tNj{J+j-J;#c`zUj<;~A*$p~Hjtg3&6iLsZ3uO2l!DD*! zNT1Uy_Q!sCzQ)WQYZE&TYRy+2vYf-;-AMfT9ex=S5+nT@2N%d`$ka2XKbpb9mSo%(%k` za;8Y0fBHfc5nHGXJ>42;azF{gT$HdRO+vVKc0DV{zX*$>04XY3OrihQrq-I zoG~KDyy%M&yz7>JMdLb4h=850NP?% z&=zwGcXV8#NB!rZteP$y#yD7KP);WH>l4Fc<3Ym06t}zB!O!s3%+ucI^hxD8yys8I zu=^JJV`L^SKQ{&U*d-Ee7kPSxn=y%h9U!-qH>2`(H^%uxCHY~~Pqs?DXJdQPNW4Nm zwRj>znQ?m<=~72*RDHu2FyKH!%Nm4pdChh0VpIq_q7mjF@UO zf9`nFvR#YNcVj6nHVK4dx9dqw;!Hdkei8<+$pfpz<<;#~h(GP*W(OT)iuPd;L}kFo zFXrUoT0I(+QwHl)ez84U#f1-5MQ|VI;LXxh;6INOG7l3cpv}uaq|z)9!gsyJfaZU6 z^s^7e>9Y9iTP()hS%R6R-E<-NfK*QoQGf9t8N+!57rwj%D|Hp&r07J#(>eoM^DAL6 z`7x1ru?yxsng*h}1^D={96rnm5d7UJ4~^?8K()^bxI36;PCA4)Pf3yQ?@e%dH4j}h z4Dj$AZOmC(&+1-UNpwHhf_C8|ywrb(h%8d0R{KNnR1W7HbNNFFBaPxyQ`wDwLg4tD zBC^1`hP-7v1#*%WD6Uja#U7ZFqrb*erJ43PFJKWky_cpbzVDfTcTbT_@lz6p@i}yc3szD5VQP40=vz~a0$a#sJ#KY=})oX{C%9Q!SyWk zY)IW4Gw?L!e0>oK=yW^}w@rP|xWzS+|FR^Z@5Fa{{M9AF^-VR9T*9rGkmEztzDU}c zaRPE%8QiH~#H#SQRTVOgBw1bvSMR%F8><0*i~eD7WhKnG83Y-=XJGHQwYYJ; zy6Ya}s47R#j=m;iyl29FyJ~#hHb%JKx00$a?h*>i^H6;82=vWT70$k{1(xr0AkU}> z8VbHb)s8FhW2P!u{?rA0Eg#aQ{5We>y>GrE*8^%e) ziK7a9cX3tlTB(VZ7b9t6x2Q0J=TAZ4jEdJg$-nRk=rv6XrW9C{b7KP-yM1Xiz}ATl zsi%{q(H*d8gA#1N9|nULualeq?r@C2E)d&ojy=OyF=gI(Sdde|%CGBWb2@+1t|2S< zZhjS~-ndTxj&jb2*$>$Ti|pyW*rjw5GasgY3&GfLlb}GUjT=U!&?=)l^yM!u!&M{? z%Pi$#{g`%qoo_*k{@kW%3Wr$(7xLk+!RLW zFSiX?-Lw;8X$IV~T?e1aLy6S|q~;qs=>7a{)brFa;^L@`A4j+0{e!8Z!sNaeVt& z3n#9=iC&@?={4d9yNXuBLaS!N#;$=I_4#O*Tt^RQUc?NuXuO?r1|1F0{G&w z=Nu@NkFQhPs5BB9l|YBRPU4z5Y2?`GRCK-aksJ$pO@ABD!^|VWxcrtm+on&?9%op_~ly@HavMw zA6))IRb92v`i6q#sL?pO`1op)HCKdRX=8}*pAOS-2}2B-Rsj#DS7LZ}3_QZAIP`ZK zlRO-N#vdE$=f^6XTk;_pcSIHLMf1^P4|kuf{>jVI+sE{GKV_atF;M??G85iVPd}$P zL7e&|tdW}q9_Qb&9U3kmHC_rA{D~od_vVll?eo#@Y$eX+Tt0=zXR!Oc9>MN|y1caV zI2s{v#}ZL_R5trc$5e950KB;j2ta_=X{|>>b+sk2NX%lJEPonCAm+bT( zHS9ocB>vJj0e_!!%(Q?&sOWl6FJ-il{|^2E-S+81!+W}<`Zkm}r0f;mQ&q%lSk6;mc~3kkNO z-kGEDZw{BgG?zlrfc6^(PmQr`KndNab8P6+5On^6%=1%MIEMZ@DHgF}cO7SGXCM!3 z45!h%E*fOP-yF(YB@ROd&iJ+^o7CL9i1w-Vq@v^=ImbT2CxXL_+qFw%<>iBnk7xn? zgb$dcSz_3`rHQP&H4C1EZX%ZkdLYeYBAvHKiO$;0yDC}co{P7I+q7&4~8br8*rwvlc^c6gahxiph8v#R)2`& z9ZhPZcFV6aKXg72Lw*R{zP|(dMYl5FH+&*X%qNhAU0Wc?{3L5I_W~U=Y(?CP?=qzp z(ctze>-syZ^`x()j@rc4lF72m=t{L0q-tX~{Pqms`#fxg!$UUwU8~)p@aATa8?z0s zc%Gtz6N>P(-(RA#f0zkd9D-Z#<`LU*oa>EC|L#FfLvVdBgcsh!=O+_!L8d9V@16() zH4(H*N{GeM_Sjt5OUA9%!#CdNL4UV68Q-}c*IMla?Mz9kogUtm zoCy>4eX&EdgSx+Z4MTN`XnJ%R{-=MBjt!WIkH6G6& z%M{SBo1IACU=BPlkV5@Ew{T168jw5q109&1=#UUjcO7yC{%rRd6kvGL9eDFe1Bz1OFgGj~$2DY=$Q9>lZTkdp>-tT92&*l3=4Yaq z+fLYPC?-7QYYcy1<-xQ@9|%ghfWlG+v@^a!^1M@=_w_mF^EF|QOMeDQp#ZGO zESNJ+2U2Iqkz@Tj{KWJ;SZXv6y;VeT`>|K-+v#7)!e|*`@iSeTwnB{m@7XEg3C&Qt z=gL!J{wo2!$`7M>V>Nlg^~xqbvFFKk210&fGq;0a0%5MT@`k&**~XbE+#K#6e)>@a z$pL93>WCGNU-5`;b+iGa(ol3rxd}W7`sy); zkw;|An{K*6HJf9ihw11cFW|jsA+%HkN{+9guRdu*(}&NHtU41n{4PS3pP4waM3tJ} zk)kgeM(K*t3E&rKK~i=J1nnj}Vbc;j`rxl8b&A>xh9REVC9jJA)#y^syBwo_! z?ELM_VezX&*G!I&yfdD z2&?;?%g0%h^-aN`E+&DdJWE!me*?~+H<`=hzhbxjwxHY)8sx_flJb?D$#d>5eDJV~ zsh(Q|bEk5iUmjOKv_;}|y)261L=DGwhe}#`4aosIzGvb>aG58)A~#X#?)$?Bi-`t0#{sTgt)WVHP}) zvd1O1D&TH00T+u-M#hj|)(>FntDEa3joNPcfzBc&vbDZLV5{#z*QDw{9NWaYG z2Wqx3iwvqU?V%AJIiA6J5POMmwk3JJOcWI)Oh6PgQ89Z1X?dZ-DsGL%I?r-Q%1L8~ zJWry*g6&XyP(j!^PmA*?R6)W>k09^S4KRLXiesgn@YXsL9Mf!oDx}4-E-MWenmOX5 zs7!d4xs-N`%J5HLdyVag{>`)pCE>lo{% z5{;3DeBf!$g|qU;FxDlM=?MNqC%vA5l~*_(>)b^A@va1vJr-hv;X1epCwOPp+blFD|60Pzx-JROL7j0!|cffXab4bf?!oVGY@S|r77K>_Q^yg4^Y05Y1uqvL3^*zAx z_0w@-rVLb{O@URv-=mAj6#VkbkBz8q)IEE%6jKg`$p>SeS8ufR(N7XCU@u2E!biSw!`jV&c_?|4vOY@@BMdC2O`vzA_ zCt~-~=_Jm;3>wXZB=fjF9n{KzOm|ZVIM+hO^TvV1usk=H6@z(4zQN)voGZ|)o!s+t zho_TtV8`+Vfxj)cR?Q{@MRpFeHm%N_gHHlBO}$Tw`*NW3z$NDEa}kgUn~%L}DkO1M z3pE}a1MR=$LE=^u8DI38ggOz@e)le_{x-rMYZi)kM&a7sZ)m{$ej=)LnFN2-q0Lq& zarw(AxZ8IKme}0ljcxL$hMCr|h{vPp)w3Y7&K%D*Bw^x%5jvXW3^ykEz@@d{h`sVd z`XjXxmmf8O)&(=Lz@ZJ>TIBfkr{lr0NCl^ybOievC%ip96tp|ViE7_OROQ@z6O|>wE?ps?yV_mgiPoE;tOKQaOekv_1vLLDRZejVj%k=&0P4HxUKFL~tS};pV0a7m& zk)W}U$o9EnSP=gdgTkU9;HMAPy^obYIm#4JYlRdxzJ;`n>tfzUR^O+LUDa##~6{R*mI6NufmG zCBy%scmdYN+VLS>nk;Doc*w2V{BmXsJAQ?#aD8Po-Qqr$ZFu>Tiq^cgym)zlmh6sa za*dy!kV3si*B&+Zb{u|?=1TT2c!Ef)-_HejmxFY4KHon5`v z19z=9A?>4{kg~K5{vDh|;&x8M;r>$c?Sm35QF5ir*7@OY1w&z7#B{Rfog$hUU4*jf zOL3Qa2ksvqgDpMv>`#FrU9X&oK^_v|Rp*at#YgdMp(6i6egyJ1+$C!bZ0Vfmk6~kj zI($2`lq6ps&n%N1Vmd*Jc$W+^3*B{?89iy#%So2IZ$p8si~_KWMT^38)=PgJdC_?S zEK|a9&PE%U<20SVxHAlvBn(&g{9sQJEfRiZl*&A-pkeN3LF}O^-(tfl9C+PDyxDAs zHoZquJf-NxWolG%tvE!eN>kxVEw~td7Umu*rpGMB`OTTzaWg-XI@e8Q9S`g$@0Xl` z>(@?kJbyi|3LK)vuPdmdzPYeA?It~(E>CLH-t&wninIQ!o|9dz#*C)EF0>u0BU-YL z$l8QxY&PWlITr)5^-V8+@f3lCBph6?|w z;^}9l=rHFqt@tg0vt6rUP`V67jkzpnP8jOA%aT(v>KNQr05vBj69>mt!n&Xj7`dnr z+|*uC-h~s?yRjFHYzK$}V*ruvYa!nEER$A}O@!4lq$_I)%$sV0dmsNrGcKQ#mXc3Q z>SXY*&s*a2V;fxRn1?2l#i%c;FmbbzaaPexsFZUi8@%U`#-e=cR@}(zY%zo@vxJOj zzaQ?)tjF+iT#of|6skrHp!KE6Fcmey5mI69>`szuR7BRSy~K!_uEJc~u{dc;Fm0Nn zg>6!OWN1SqZAmaaGrwS$Lgp-(-vDWF+y@(@^S^7X%d+COEnHHt2{~kiMgl z_`&KZE;2X>ABE|dIL3tQe~I&t#hxKf_YUBrjqizZf(_P3MPkMvx71GbXtl;K9%wph zLe^#(;eD|<##w0_TzlS0`imXWa3q;#&x(dEG8$N=KTOM3&4saC&R}*>ERmb^1JATh zgbyaaQA3HrVDq=6$Rvb@?>~qa=beVw22n;oNE;L*3?X!2HHe;_0wo1|aJfI_*RK~-wXZ7Ub+kd3#UML)@uf#*Q6(G6D z5ThLKgX{ocVA4EPA8$+lWNfk2Ke(GJRm_6!RXQwkecs?L5u6id2Sn5_$H1;U64KL7 zU(zi&r|&!c{rdwNN7h66(Owj(lAyC^C!pFQHCPmUfxH;s1g#qf+0WmjiRgPnBKa+e z?A`bQvy1F#cf)JKcqTGuy1YpK{d?@e_IOa@OJSVSUM8+z1GUx!P}g;0U|M+rKb|zj zSh08HK+h-oUwbZ&T;=?4_N}n#vmQFya;)khCywPUqjxkDAotjRWZ=~bkQf{bj#<0# zuwfpQmxtq&(J83>Nfa&&kFVZ6Clb{UNueg^;e0&*B{kE1gx2m=)c*Qu+T~CL;`N=( ztkY4{O9sH2O=T9poD6o~Wm!Ww6BfvDacC-!v0V z(+bFWJ3p#^kHPJQd`w>bjy@kQhvN^G(eQ~gaq{Wo*3iaMyOIFGxuudYx$F|}=J)-O z$!5^^ZhDZraWmN6*Fo>YmuX{29wcUs&}U+ym>?>`U;phSxjm-?PE71TrxnBG*_$Jn z@_j8lj62W7Y5iilUhK!DePP({(L|o?UCn=L7)~dMnu5sD2V|wDD$M3y$X}0Fp#8@V z5;3IVtBTpv4viOzoWwIaizVZOSVF_~Q)e)N2(FP0r?Z9<y zsP2aa%L>S+ZwJ9Sl+P@wlfb{O3_93~lQ&P@iNmE@MsHjuNpgHoCYMfvxNl$RNbPbo zw9kfT31Re%j45i=wV_D47D}@V;fl;1sYMu%n?<2% z*$lEj%nZjBeuXh?DtuQ|2aj$&dXBUa!$g@y-D(%e+9Zh?LW==r~&B+Du0~Ik*4HCv1MyJiN7L9?>s5i@Ro5;y*uK zlq)l#n~yky?)W(Xt=yS1ybC3d>cI@lBD(6eDwz_#7Cz@D!e5_Ryu`$J@L&6Rl#rsN z{aiA2+$~_IT2_!&Up2bRC5U;G%6Z{@r(osJ7v#HP0mLto#vYr=BvdAa$Qy^kY3(p* zJE}zoRyo0I4*^kgVWA{37H3K7oKkIlRcXk=msvLFPwls$x zX`KU4COTr;sW|G}Yz@u$YwJK|<7DQo zi6cH;V~6KHN<;VJXFTim`S@ei9lCMj6IyuN6IVVy%|xokVWF}TF1#H;{@am*9{vC5 z$;UC!S96hx8>j|(n-@5DrZ|YH2ZA1xO}8varhVMJ`0#~zx<2b9{d(#LUCKF6s@J-+ zf1d7z1|@6M|8B)jaQ{P99*Dv|lRtFb95s3(=_{VCaUyoZG0?)Xnvce)6Pa2`=G99V zsy!nGcU{>Jvya*1)(CFaXL^MMDCQBn&B~~2J3`(!nSjz|GYo#1fKBR2n6tNpe$f%2 z$Z7}pkhd5vmHx$zV%9^Rf<( zP5sV<8l8Z!3#oL?d27m0>_fk6^B^2T|CZOXl=8!Q4lhw7x%ycHcZj zgE-DDlGUQNk2A?%^(DNr6%qLP3->(|%3wm76)@pTaammqN^yCa$j!2N;(ILl z@Rs1?giEwn?thBT!=K9cjpGrDjErm|DoTTfoaerd1`#D?q-aQ6X?)WVC3_VSQHZQa zGP2HdUq@3E+EFsnQcAK?lHc?D6VB^A=Q;O%U7yeU-GYarn2*cWX2kaZa3Ety&l(xHyYm?8Tm3$Zm%in~h+Z9lC zs~FSbl8q5_?$aZlMm&eya1_8l*0^aaOdqvEjiP9hBASUSl>)JAMI$Mi9*=7d|0a1Y zHT36tXISyf2=ssEkwI>EctHL($Uo$`Wok#rIE$UIJI)V9MHK0~ee1D!tQ!ty=nCIC zO~aRKJz22lSWXdqzN(Zym^>@yN8gqd>hO#?k6Sl3{$~h&g9B(C7X?qYtAg&M2`JvT zxzb6?8u(+B!6<(Uzt!g**q-DV+v2xyig1-8kyR3?1*P5L-3c*-d&P8h7l-xNGexaX?? zj;*x?_s?75+}Jm`V99B`c9jouT3%RuRTArEC*mKYWkl4chvfAr!oHzgx}r=TmQAkY zIZg3|+HQGaPn!ilKz9?A`Fub-2XEn&(tFGwg;CO}{fjAglp(V3#PR)RWsv)~f~tO< zM)r05#=^1DP=6wWIsfMxG)hMZDs#Bq$yaYwnimNfQDHFMI1LI%_TfC8cv!Z~o2W^; z;!(U!o{PL7ec_4_zodysPn(IOYjn|gSdBJiKO+IdCZtq*7m6MA#&!Nag!izE5&U|D z(z871sPa#`-6IW3*B`)rO|QtKiDPJM`wHTmwjAu#kD{^RRh$+-4{x+h$J;}B_^zHS z?W*mE!Z*i=cp{LYP6ITs`AnZ$aBhzf0sMjvCS^_lZGW!@VB>*L;xZ6jx8s+NSk!Hk zVlTf~2g0XC5R-Eqc{)E?$KF8dKjsED2c1PsT*k1mg;dfMu;=LxGA_x9yxddH2ttFo zu05B0{&j@XHEx@UDNUlybW`YviN%7`Aq(3Pa6K=HAm{Z(caDoU*bw-f5 z&3D3&;4aq}SSlnNXP#Fg5)|e=Z226CAhGea)JXyk;FP z;AT9%U!|eLl zM^}+Wk`44!%_n$&XEo>Sd_Yewl;L}F{X&V2P3U>J4c2x|6wa<*jMq(4X<@S!{-lZg zX(kVG`rRE=cG6n@!|pNg#k_|6r!f;(%2N_7U-ayV~UC*G1$z(w3L=~UPQoO!B^`KT4f zzb@(qipdNLzv+Yi`P0-`>%r%e`2a>DR@UGl{H4KDZnLkbRTz{s|0`gHmt+&(bK8i^LdT#yj#d7-%Dd08DE0^>sV>`4ve@y4^EFbqBgwZ&Tz}A zY}GlGa_FMUsSoIalmy0pq7;+AVmcNbX{L%duVQ8KKH6szMsn7a(Qnn1k&IUWi-Y;Z zC*U{@%HX_K%Z|gej%0WxqD0S5l_0w1<7vjFEBHVuluVH_5*l6!Lf4a0Onb^AOz2D? zyB#=hsmVq5*x&**y}5z@@*Df-Qhj>RpR$V!cER)cYvJ`}z_M*|PWfH3B#+4XHk z;Pm+!Na;rL9{e3;R?KGjoxbxq-mVn{ad*tzzgM9ufWcJ1=WJ{AQjYa&0=GKN$<}aF z67u66bC93Mn<{sWbVfX&YhE6pW6Rr6XPr3Sc^*hoEURda+I~DgMVzlQ#|+HpOVapc zP14%F7#<%wNoP1+riFXI(h9y0dHK1CUiq1ijupu9qnE*FF9l>ABT3$yrF1AX03YRt z2^f>zVEHzHU1F4h8yCrANkt7#(CnrgxXgUnQ8!#sIuj%s{xGJZ8}W==2>pu@Xt~7- zlq4_l4*k)`=D(BZ=fBeMt8gp^ogK?;zN>}rmV{v%$Iuox55hT5eDPe)X&N?ijZ+(j zlHUuig8lr(ur#@i7H>SnRJ)%C%V_TXwoHTM#Ws*$f7 z2{7yLT6mP10RP%n;_owC=~v%ma+fy zcedQb5cb6>;uJ+?+;lyHMr@u0hdb)XrZd~%S~thfdHV~e$xh+(y$*9%8fAR8a}r!V zF_k@T-wgXXZhPvz%jC18GP%!ViGTS+GH4hG<#Ev@P-_#)?7ac*ndNBy!jN%?+okmHw>?U#lK_F z_pU9Oc`BTGD$k&;A9m5><0avH&r|6=-1UHo}L@?1x9rB)s?l9!T1QyE%xG5r&3ohjI|nJNpbag|3IbY;Wvm?~wCfK7&)}5EHj&27cUp z1V(aW;cnJXcxt3g-#$DK(b>x(GP{d*MSP{@V;<48U*h1=oR1-f&zWNeN2s9ENO&#P z8UH?!6{a10Nsd5UXT zPh!l;5p{#)PG!L%DD9Y{9Joe+`BSmn#$r!hNSPFN~ zSdpvSmZFb!7HUfjK;5xYxRt?qs26O+d)p!~U`s4MX6y0#jUqbl?ImI)Ua&x-TDn=7mtBj==c1ZCnY;x$Tyd>y%4afaUEb;!#BA9BojyB3# z;=R^&kTE`$d@2~EozVmI;+hd6{aOvTwEBVXWqbIu`6Joe_yV_{n1r2$5qMtaJdVyq zc$v6?TxgGmpF;sKh3mAx4+y6IQr`G#-v~KqR}Ou0dkFhUfC)2vX~eW)`qfiNTN^~s zUTXzxiL0a1bS!FgPKPO{mVrUuXB;qCF4XKahKc(O`DIh1&{@L^#qGLrf1VvyKh9vc zU%UyI?8nmeDGDHeQwsbo4$3&Y-U33V~}}J}cgSmaLe%4fXXez#%$aOV|SE8a(K#_7NZi&Ey@&KdMu+#Aw}9K-UN2kVi2pW~~T zfkf0nYGAm8q&KaG+?{WDFGZH2^t2LO9XXYFA6Z8-{@V)audJv}2_I@-P9WlCE6K0? zO0KJZ1~t<)>ErurF>%gZu&mUleZA{oxd->1|2>37H6r|QKQs9KW;N-T=gxKBe`r>$ z4$AC~#f;1HTrT)EPApr6-}YFb>eNNhaO*OW)x1Ne_aq9|C2ZnvSMemR=^~iC!kn8V z9)MFfVn9zk4>Ies+4C<}VeI?-=P+b)9#K&bBl77V$ph=faBWT>J-AJq<9ydc+?2JLdi@HoLOKFN zT%IsWSD)gLz6*2Wm=zi3m@saau7P@-1x{T2otVekz_?yj?7I4jxMtd;?8h|HrnwO| zSNY)8N;`B5`$W3+PLZECMoIM7ZK%<_9j-i6ps{z?Lx9Qw^6IGquqFpWT`vHO~2G;OdTSfv10dXTrLrSeJf+o?#nd#rKlFwpocCS{z|G_zEO*c zRamZ5K-GP`=($u&C<{5q3M&PiCo73Q|E0uKZfKzK*<~allI5*5EW#^hr(ul#L%P(? z1`ow#>#1m3M=+2Is-=E|>cX zdmnx*Q2!s6{W|lm~$Tq@45`(QZvDJb~r4{>PE}O zpRvD&`>m9(#tp}7>0{1KrGEE1x1ZO*j_epzcrur+S*ZdaUN_McwG>)3&Jz4`TtRL0 z3prfJr~FA(RA})ZPAS<>^VZf=_o_si)pD8~t^7>X&VeRW9c4#O>P!b!oKF(Bw$0C)WQ z@QleX)+4ozXj)pqSe327!KA2Jr#k5 zI{ruk1YZ`Q+vUSlV6{I6v**J>aMewHxz<{AYX7k+5*FP=kZE< z1a6{X$ew8;E>b(7Jf?|bbiAgf?qk4CViJ^E?I*?>yP0t}pV2|DOR#8E9gD)$;fF{A zG_F&}h6`ha<5j+peIuHvpjQm4V#dUMc?c3&Z(x0XQ)OuhEX=f|14kCzrD# z@5^w(w|qFHGFy0b?+7V!41!UolPFV{1}5X3VWfO4-#cnA<82`i+j^T(<5&mXe^4Cu zsJ^E6_?7g))-`yi*A{;-P=SZlvCjvYxwJ{~+^Jl<97bGobWU4RV98;ckr&BxAM-F0>vm#K(0s)Z9-Gdvr* z?K%FJgbi#by)Z{t4&H>S;DP5CQGe%C+N>}S&0cE~p;shK{Q8{^eVIycKG}c=PUTW_ zqbK-N>MBU~MqsFV8x8$W5)6)*1OK-zOuUb1z%l(+%{#-y{7R(@=DG+||MO)^dh;;p z%TxMbLp@pkWf#Um3YuTY#q{ti#2Pt9+Myl*`{n*{)dvrhD;pNM2c3B3kPm@@`KVncI($8GoTt$s+ z9B7_|0e(6nLRULoL|2z6l-sQVUwXciW8+qmQ}?!$m-98j885?uxHDz)H^0#48=FvN zi9PdCw-`={AFdRr%RqCtB8(ZQMd$st#PJXQ&##<{;&q;IZh?>`gNjsp_9;>j9gCeg z*I{c;D7^is19GiA?k<*2=Q&?MsmM0YQ+SL{&74cGPW6QGk00<>{gi;)*C$ewwU04u z*9>6q?nJFq8KC)2OgI_eVUrc-TrD=m#fe#L;Dm<|`fvpa^$frwMGGuissmZoJMgt) z9yYdpqE264k+WlO;M~%yRQ-Nb46)q%>A6>xFAUdn+)S=g`PGHGyc@# zLLhFD&P450b2QdjMyi8vld;`?pxU$@onyqwuAoL%WWNGWw!9xA@+Om&>!*R#%+|Tjrvsncu`gR9-{@9kLaU8ol+L^>UevlS_-bion z+(BM_bVNa)F7{=ZWA2_H$X|C8U)C$)q8eQko)clZV+ugy;zOdkdpmV_JBjz>TrF7j zUckJgB3!J!fR2~(rF&<5pgz+RX}W_yKWtbGr9b9T9_PINA*BY9=jWrHk|rbm>>FLV z@|W2v!twHkRB5Mv2JT-Lj(*)|v4^u@F5^7IxfA4}o$K{%F#AYUijC1PpafTJ@Ih8- zHXip#CTvR-PT3?vGZ&lTTIGB2J+=z$>k0U3sk4W7q`957pn!hN+ zx+E6<+KfT>OZ}uNsu_|NMR9pq7AGq-sH7n^y z?yf9f)=LM=6==DpDr1lrNALdr2{x-EK{wqq8+jEj+EUX(pvyxB-H+Vs(ESDjh? z#y~J$=SlN@#NgoCA8cr^Eu9k-0bN>Oh|7&B_#R(6GCO=Xy6 zOB^BW9>HCiv2$=&9bPylydcJ$;sLZOW!|Q#iDd`wpDz zttvDTzMupC>gd?qMf*2WTFr51{7j+FOJl&wXMJU8rX5RKsb`jzcKl zFdn_Xce5ca7fC?PJa{?bDYd;gos4t8LZo}dXuH@?+EiTvLCYU9b?efIM~54p`6oh_ zuS%fH@>4lah9RKFXX?^w42^lWxIIA%84%9~pPFE5w=@hI0zVN=4_7YdV2oXfqIA`% zHqvHf3pU?^sryVVZf0?(^4o-?7-TmMC3Xzc*%k|Niq3x!8Rw7X(Gu9AZGy`mbPDd7 z&V&a8YM^@SIwSPG3~SZ2@x}ZE7>?Buen0jM`CW%`XNV5je=3xWEV@PWn^$sPGjo{B zJ*&GtpJLj`ecnXr*R;LbkmDsD!1Nyq^h#eIeLeJ@+*BN3&%V!v4G|Hrb=x4jj5`bZ zjGcgGzq{$)np)Ic;Dt6d*Vwjl4IEP+fD^tZU=)`}T=*=9k$?3MHWx->s<63wHtacjE|o%~A^Hk%xV<|uI*sh`QZCHJ9R&vClM zCY%JnjwbtN?#EZ(m*aE;9bto{H4S^0h1Xn^Kv`D<)8bCtoRCY_9Y zj)dTvad7qz!lLn3%Qx`b~JsDozjK3aMG13FHcit;*( zxw*bGNiZ`LO24^5c2tW&6rRBwqj4~lkpr0nHSpZCkjj^e^Abcn;acuhEVVS|Kl98% z`v*^e|DpuEZ4u z^ye(qs1K!2f9|D$Sq6AeeUQjT9pj7RE;64mxcMxXZPa#xg{@^!z;zhAZ(95(0Bg8hsi#hS|XEYar$F{JLvJ5v)5253n z;buryjkr?sz#7RC2OKZZw>97hB% zj9;<>o*~od7EO~X#4)j32K!&%AimiQQMkadeU7`a&BO&OCMPp@@FzYV+zFXs#q_3u zGLcT7jptrOz{XG^yopL9!{_CpYG(rT`HwhW56LFwl}qrcX#}3uI*EzLeh^!~JuvZO zAK9N6jn0;f!D5j%`(MEx%oep}uQ=`n#}|kTGgq;Vp&MyVup<22wh^P-CqmMWokYof zDGE%AsLzaf=&^qV{JOOdPrDi+4JjovQGql(Jq$p5=@a`R81p`tOqPvil+JVzPpdn4 z^rji4W*DN^!X%<6rAIuzK4i!BFM_8Edx4BU%c$R|U__P;(4CL&kzUal!qXli3Y=r& zbnYN^6OW>2ud?L9&(q{O=R{~3r-#KWAE4Ka`Pj;<e__dJ8gCD3ylnpjAui4LaF6^u&9rVB5he^F$4!KGV=;W85*@vkt&AKU% z-#&3(McW)a&)O79kxK)**KeRgay!+c-LOkwxoB(5)C ztnvcQZ57b{cL=?`qlXUnT63JPNcuU~77Cth#rb?)$QAoT%Bw8lRoq;-DLPbHYF5Xa zt#J<{Pdp~aCD$`&L;J`i&t$L-Plec?MWjjkHpyuI!usw#2E=JBB%hJMpBE31f!Qt? z|23b-U+V$JvJ1geuY+gL1kkvb^I_=tGwQJCFD%S0WWu~p;J%(JqP_DYw!JK%KOe@B z(jSwsFg=xOf1p(Lbu)|_%JA#MUXWW7zj3E+8&%IT!=bX@bkdqwvO)PQZHPSrUiRg< zOmqU@d{G$rVi8A#_Oh5wqro6t7sW<7pQJ6r9zSP9Kc0O;W4FYE@oy*mZqtoMbvNiC zy%#v+ZB3573Ur50H+hfS{ z57%+u;7wTMa3A}s5xT@g!P{z`pmkvaUDtk>JWcCm|0dsOU$^GNi2;A6-9-c|-6vzt zudQ(S=TbQFc`5al=^(%RouTZ-6;gL4jmyoxA$7O=nKR4{I@x~}Zb^5e6+{{ar6f7$ zQwy5@@qxO(dTc{a4VB~eUJF%Z+3Y({$f*~nQQ5AMK2nqxo+|Eu&>1s%c1ka4+iP(g zzaT&$(NG5O7v$rXp4FTi-Im-~Xo*3Jag14T7!70W!G7m$_WSI9^7oTB2!`jwV8Rr7 zpyabaY?6g=P^EyH#FxU6DIsKBQzNT*UqmR+Yr$bvQ>?W~A%cH@X@-j_&a>Y_y8fu( zi=hU%tpE$^m|pEkn1 z1N%Y!Vi@U8OoTNy2bt*UkBB1o->d#Wkz6ra4k?l>`w-_~N$+QL{asI%#>ogxuQtHm z12LF$IuMTi+>Ae)H!+`!I49P6j^p3F7e-czlGWupXe^($s98$_519mjbf*HWPK>4B z*LcFx*@q$DGJ$c-tfP7+`iw=$TDTi>n$3)UjA_X++;>t>bhce)w%S|L2LZA0RbmX~ zS9Ab{T3jzM!0Z)jI4$)yddkcf{t+{wf$`2L)*8SJmv6YY6LNq>8&70cNNuUrB3U#zMm3WJuI1L3XbTx-2jh{MA%M>+jBH<+pB< z)XsdkaqkpLeRxUNHsz6JPK)Vj4O`mzPFg6s>kaChU(f#CUW~&hRAFL%4>@9RjNkos zUS&!eZGnuy5T>X{ zf&XFZahyZ~=m2*fy0F@cOybGo=lDX}#JS@%{(U0XV+Tm*TQ3Mo{=@#rJWD?`A1BNu z3E^zHN?4WV58|qdsG~Ch4^%&69u9DsbEk2*)YuYxI#l6~v@*X{;S9XLuY#uBZ10Zo zJ}k4l0%u=Ik|lw6uyJfYS#jwy+0@=k-g|4&7CTd%)iQ;ClG;sBN)B8Wq_Dwdo8f^^ z4E&L=MOT&iXxVTWi%O57=(YzWC75$Z?^+KH4sy6>+-kvF7)r}F+<>j&HB4fl7CAEI4Dmkei&w^~ zL#6TtqA<6IF67<=BN^$SYCn@vlMM!~3uYLhpH1b(xn0Upb?D@ouxb+~2+McOrshc} z$qEI3R(PzBOxaWjzduBoeH61o=20z9-c*h?F*hOoxHIa-u#o@!Ca7-4=|m^0Wc?MEMs`;8d zH?xdef=;=sh?FzfFsQQ`tG_=0))DxM1+)2!+cAE~43KQRd=8dHy7iL+nsQ5B<41 z90ILE@n>~1Yv;hd_a{8Gqf3_1w8- z54{#1M5LC^fc*&v_^xj`4^Xl;+?v9%g&+42d)qyj(wu@}8~!7ZpoRxLBiNa_z9=hg zK>V@{F_oJ=>Uujc`gn(TSHc2@g4EH#IfKdRwE;n9Hl9#bC8L{?xSXmubU`C=`F9L7 z*Gr>G%L0^$wqx|Bj1vxi5hG2XS<-uaHI%i#LH{mk2>oSK-`gfiDr@DgU z5$eH6V*z@sJOn|GNnH2iAbsU_8P>;yp{~hB{CC-x`L16_p44aI(L4)S@;4OvzN%yB z?p*ll<^{(?G?==GNg!XXj&GxKcooWNv}M(56cl_Vm70|_#f$44+Qs0)gfkf4x}J$m z`c2H{^wWErq{*^BDXR@#_uAgV zt;{2ODOr^t@30S7EDfVBK@{pvhIY-XzHDF&4tl|92EhMfn>Yrz?D5JUD&*hTap-%afOb3! z#*?9O=yWRrR!8p!!^4`m%O(mM`)f!{Mh(20st%%_t>nPMU&Lus5ormXh7KS41k;`O zK}AXwvpDbp_Vw?g%TIDX*uPy^?y#EGcUTXmESJr=|A$p+2P{e-2c4%aFo(?`0fzC>gB3ku2bHgRInTR|^MTthZ@KU=NVwytwnR;3WO9C5$%jT z-p~FhDt;rHC^`-a3LIxZ%d1Sh>JcD+WCFPlB?UO0agm zGu}2VVQ(jICASW{LyTn-omS8BEaVmV5|?9m`<#;mHw|k@h|5)ypzs~u`z?Z~1rMn8 z;g@9n%*7}->n-&t>mr)m49&oeVIAzExFXRRsw_1TR5~axAoM(*50N0zo44ROZh!2T zumz8|ltaGiG~!;}$W&#^@;ihd2|+u|eW#4uj?KU$B^w!u+eR?#J4nzz8{f)JfVcL~ zNQ7@C%#56X-!&)WKW?t2kR@b9*G_@6U_(ag)+krwRgyauLPtDrP3iEJ+WL2V`1 z3Rb@KM~yi_WJv?J1&G$?cItz)#6K8ZMw{q5m+<PaNb5R9 zBi&>$xa1T5CuPNm2F@a}4=*u8U8*or8;AL~IA@H08l+zKhx*0>A}Brwd&8!YTQyl^ zB5@+BbwRW7{U7w^V}ZHwuh-|g7u+eG)v zBoe3M)2zEe2brI2PL9NXMXP7`1)s`S@Ox$+n^lDcOY}%Z z&@=L+`8>LoE`sqfJ%pS+hEKWdT&uMa1}Fcb-?+@F{8BMYJinNJk(dr{!gVu8T6N&K$FN}I>f$yir(b=O~I2RKUZPl>B;WD}FzYu598uC{n0o#4j z!7^NytTmD*Lj{kR!{`Jno^wu-|FrqX#T7BD+7$%9rlESdG#OEh!1QyO!2e{29(NYO z^;v#cr*It)avcK6iH4|F`i{Og^~aa1w?WvmHDDpjlF(iHAd7uOt!_H#B;O;9nGwvn zBa5cGws5x+h}f_uO&E=b?|IYl(X?e~?-2?YZu*eVhK`tGuY`9>gFz)mn@n8wl|D6J zf)U4a1j%MbIO9+O4A0d=!$?I;am&VM`&Z)H$>Z^LuOIRJAx2F69dVYXjk{cqFX( zKyigrGIR7}9XYvJ53VIh5cul}f2L%>+6@}0<{~0Iu&;xB60J8|q(@%3Ob%8iO=Nzv=-KzzSxt)wwg&2&Vl!&DcTsG{R z8(4~$fL_2LnfPrCUMvWu>Z!hT)BY`y7kggUBpbX>aNh2XP~KXBG36p4%e^zcyy3X(-bQfw zX*+atMc<^X-wdxPl&PJ)9Zp7=V@8TA91<+VB)dGm{TgNA#zSH3^hezOX~Qr2`icmd zw@enl?R|t{LER+#eJ52a4IpQ4Cm^hz$R?)LK;6PRi2BH}tuKV5>qbTJ)#Lamg=gS{ zRsr*dIfEzaPr|aK1sE-LnVYN6#%(?dD7u8;j^JY$nf(xB*ao_|G@Cq`(MjtS>QF?x z8oQU?Wz7}F@M#eDJJ#bkNHb2t+`-3$cY7z~9a<@z+9hP2{r%bS)J9~-HEhJ8)EjA}KU8#Ms_PX~zRXe}v`eo5m?-htB` zp`hV}9#)=Qz=#H&=lTJS@Pq5E6~-T8lcoPu$WK}iB}EeG$nCrS^xmS{627Rz_7bgs z?`cjB4}E_O;*Xi*`ChlPaK!pB1nXP@k@+1o_mKjB_NJ?N+&C1?K09FJw5yzFA_#Z3 zf1*DOUa=qSjQHt)yuixj9((+`91h&iML(km7~8mr?)TIXT68?-)eXKSdtPmarP<3- zH?azLt!N=jJJnHNZw>3UJ_%>&KB1FF+Of%ND&TY_;a3|?lEL?fv4>bo%DHB2=ybs=OMHK@kL%zN#jw`9BZ+``Y?GZUTuBHS_6(wQkf)g;m^BTQZ=|(p05`#%eo6vPk z85aGy#ZC%Y3^FCY`0>>c4lFg~1#PLNGC9qpq4@*>>1_@;;QC zhT=zzfeE$=XjZiaf9G4`NdFr~ZpvcFc^)YcH?xLIRXG-w?vf7Osm8&qA?GlhI7yNC9TKf15)^Gt}XP3-@&Nvn`D~$O)`b+ z;w~K)TXhLMYAq zv=csA1tR~X2mUza43kVYgXO9!D&OA<6EB!B^UkfpY@1R15H(p?=`az#bgaZ5zm%b) zRSWq4axty+EXsFT(C(?z$z18*^ud8zdd|0&C4pm1zU)}?xJVeC^b zw^Gyk917=NK`i9lVwQtwf&L zTjq1YPBct4`*r_BA+fBbwEoBRkANh~|cK`*S={ATJ{l><3-+|oG%CHwRo#eI&-_o@`{!92_IH z8l!BiQ9j{0S#fS5mVZ~^xXVjWLx+b4CI7I?V~RoM4A7nW455c3AM$kUbP+k{QC$J}b2iO#blFnYNoX`gqW z4RSYu)5ctn?B81a2PbJ7H)p;c^c83Ce2v{V3W!(OTzp||K~Ag{Kvx#WI#{9w=80cn za~Zd16IVfz*WX$FZaz8sGZ@bvy^ou0$AaX0C;nGy&atd)h)Vk1Fu7GxIAr~gp8LTk zZ16s2%FD^T0{I)D)hJ7=^7n%7q8RQ!_B7SAGz%wTLsI_G(G{=U( z#WgR%J|`X;L<{J&wh|_~s0(toM4^3(HN7qsh<2S(puJcI#j9F3?1JnM*k*Wt=9&<+w)y+oQtgFtpymlb(TzVlg<21(}&SJGgbD@%Xig%dp zC@LC`hgXc4b<DS2;FW2!?iSC5z+_z(8>g4R7wFQu?zX zbw1~mcixJY(`bTG&WGuJ-Sk7#4UC<}vFk@JLf!NIAigq`ez^Y#)5&D!mRGPq^y>ty zJz&T2fr9B;r2y1C@{(C_Mhg1Q_K-y@fRV~P2SI=P5mxBw+#t^99t z+c5?VylmkY?<)D(S`CUh&P3NtYlfVY%$d!Z~1Vc#7=8 z8|b@g5#Eb8V3)s&#M^uR!`MqsV3??1`SN8O@mbsZJ!qbw;qC+bvbt)-cxE zAK*w`5UK|ghe89bhEu+ocjTx{pxx|R%68ZTqqs<+rMF@ZW*%Lz5+ z{if@-&I2nV0v2-;=*n|BMCJ|;Hmnq*Jh*}8{co9Chey1zuPzWxZf2O0W`hT&`E$Ie zlk}gI7Zgd?Lel+%;Cd&EW|FxEh8C z-05_K4KQ)n5LH#!h7WQb!Nw&U&FAhWCLtkk!2dCXN}T6dH)nB2Zx=}%R)Vea=CsSg zmQ1g=1y#pO?1Zi3IEKu5y8dSXJ_$cUGouoT=SMxb++Trb>)vC5P7;p1{!X?#IpY_X zGh{>WOYE1O8)wYgRo}a4rcRMKUmK%hS{H**kOcxLNu>;=-*D=@&UatS6>P*9G zdZRwvph0PnB83!b(11#H_OnhIN^?RY6zNY?h(v}c%_$T`Dxwe?C>r*&wn_hiy}nu)qFMRdb1p!xXy?9+FfMH z&RCQwm7>cHdAGs#YWBRtE?WiJ(+LtCQ@G&RVBH0~d!9&_TMJjNcf99PgcK~hlv zvJ#Ew=~06;2~vC6k#wu`Spj*TyYdZSb>CwuI8lwUn@+RRxmU^3vJJ3*K$dL2tIDX= zR@1$|K7#r0^%#6!7VT=+l2U~@l)ZnJxW{xbmOaMk+vtex&l_Qf+h}f$%3QSnHWAGa zj)7aA30Ub%neldeFqqE-P4hWQYz_I(pZ-%?GtnR3tubb&CpOVNnMa8K^)T41D_?O; z=Lrp3cLJ1jV{ygYbao@()6+M23a1y{gQ9|SWZAA|V4FP?a~xw~)t|NW^6~+}pN>ut zhjdahWgcyeoQhrcJ$NDTGO^<4PJWKPRM}k(L=0wwf7Hba`->7-F8`0}eR9P#k@s}r z$Gh~Ck^|G_$WfIlAd4;4h|X&xn%`B(NT2fq)zV1T%TgaYcn`FMRt8r0m(%G*gUsnQ zCVYk>o%|I{z%`{MwD5fdE+{c#bcOEt#Z`&t%5aR2O*{E8W)sV8_*XIV<}Njskw>%Q~ zWE4vpU@8P@MB#n`{7s#POq2k;CugH(YazR1d^#>2zK&ymEQhn(zLK{$m(eq&znE}S zYhtu23AQ-@qcxlU!`CZ}>0N^)tom6>#V%A)$6ha-shCGbf*zpb@4r+gp^F^jIVWjV z>v-RV54$!%3$l%Q?^^-yw3?p^-1BcV#PtNT?N=*Xo|%aA-7drSi)UckENd_*5rtcJ zNysPFSe<-j#?(@je2q@PYT0+-xXA+pWonrxz0G9s;At9J>`%{ZK8NP}rxJt3YGm6E zjtpC@#6A1Wacp9Sh5dm`*xRd5x9f1kRoVbP@XUnk(>2K7z687#Y6H{K^kGeJJ>9Be z1Jn1vArEhL!M|gB@%E`Fcy!4R@;c5QYa*Ax{_2kMmr)j={YnTw%g+-zKX=2D zLyPg;Pfa@Z(d&w>?dxGwxjn`OU92ehaU0$kHsicx2Lbfg!Lhtl=j9=q` zR?p7i?(k&Nxpg|9rCG&z3+CXu%C{IL)<*U8=aCfNty6nnKva#oVDY0$vgEcOB$V#O z8m@=zey0lq-#^p0Ka;TUd@gNTt&dmkJCPS6V>!jCW8u413GZQX!}*K95bKd>cGE#I zn1ACnTf(d&Ki4&}J_%0DwR2)@p5IoOR5?<9{=_?SEB+RW?O%lr9p6AE(jfChVj^=`*dc$ zC^>aBg2-tZZ+>)gD!>ddOR<5j_F*lNMoHXfBN@FhJ45GDy8%$zJoc zrb;_P=ubm0qO7$AU)-)Dfo<0a9bQfU&00)*BXuz)Kmk*lc4F$?3aZ?Amj)OM(b=ql zq#av|p@tSBkX7*$ ze^riS9ld@r>5ZZE&M6h)8j&?{%q^7uzOV#!oS)M`dw;5vFx@h8lmh5|_$qiXJRWsi zDrl;=1$?sL`z*l&^!W)>@aYqP$s7jWb+zE$QAI?n=RIAqrkAZyI>IXPJ}H$ayb1BG zGwok%0TXMIVaeStvOLld+3;s%JfBgDICh$?ZF)==)<2@76Xv6iNDNuEb1qeCyv-Ol z9wg24Vri#o6?ZzubvG*I%S|f z^|hsBk}nhU-JG7k7KizdKQP)$%X8L`6j6W9DL1@M0LM zm;h1fH%XUOJ_)Z+gXfC~L@HYdMSZ?O+3SNs_ZjnG%bHxF_ZnGlQSe_dQ0pf;fyLlt zDkGdKKM3|iCYUX^Pq<0cTv$%R1zL*tp|^LLuxpYpZ05QB_J7AxE-zG=WV8Z;_#FF= zJKKfF=Sv7Br;Bka9_qpk-B*dy?ok-lkPGMjq{Fq|xx%oP4B<>wcSx!~53eqUk}Hd4 zg+{u4pj>hkKIyL!CU?5Smm%Jblbk8snH?_lU8#b0i7~=;z0ELqKtw2ZX^qga+fw-K z^9;FF)4LSyCt-Tg!iMg4r}W@}OAqCJ7#<88SO z67IAzej(|+x0=U2Kfv>^o$tt5@w#(JTczdhZ9mCT$am+`LbNt!qu4HNYQ=+u7}`ddZ0o~OPT9{ikk zwu&*n>JxExx+=_K@5A{|iIy`?^WD9SO}rCx8tneF4HKu9;^Xw^RCJXXcX(DPv_4+L zGt$M-hDfp^t0Yp%=in*$Br=}#{IW&Ko%uAV=osudriT9BtAIKOfQI^8 zoUN;leI2<>Xq+MYW7Qw3v4!_%xAUFQh;2-N-7v1-n8;_5YiUsU8CYPHg8O=tF|(uu z$JA!Pj@_QHdcCgj(0y?(*{P7KiV%v>A`$|x@abzSUis_DZL1)rc$8%`XHNVum+>f#n3w3)RQCi86ZQg|{6=;MN1 z{4}4XGM{CHJ3pnZA+LC02-YbmJbKEHWxHAUyC-NQ0JNsb9HFM%KG8^q* zwBvH}ni0%M0rfR!*=cFADvQO<+uPbj3K9a4}HR*pB) zQfboe0PGjLOP8K*B<5LZ7$(N&_YSu3PMb_HYi(slO{-)~9`&-FkII;Lx;!stfepB< z)qvc&JS(?kIy~Ct0bYMcbB|i}p!j_gTeWe3{iR_}x>u{Sd%Ordu{l!Fd^?ZS#bw|e ziSe+eNt|x#RKRoH-ej|f3`S)8Fn6Ei(LWD0v8V7BGj5g`ZFrc1`omTHUPpvWG2>uw zdp&88wZkaYTZHXS#vL=3()d&1T&Ha}8RrHV%j_l=JZoCBX%ct*F^5J(#h0||pcgKo*K6(rGSM4Dof;Rm5 z*PI*bv4!%ZmT;&;f}Edyjp+YA1Xe;17@TTPYJ!lK>9s3p^f`>BC$P|if_)xt=r8oBy2T!W^3Uun=`QYiH@-OO$XH7O%pih&%vso z0I)lh%4c;%=}DJ3T#MdtQ`4LA6Bm#jnVYbq@H*bxQ-blz>!51G44M>v7^CYBqUf0Y zpe}iXoT(b92vWRF4OWH-Eti(SfT;=h@2wvu?9dRrOPmkYB0p*F`~e{jx><|LkS>} zQV-HO;=(E6&tUE0dpPGL&-wIBV}E5!3(vm$gzFy{lGWj|pr~{nqC<5l${xUrME4h}z?U2|de}sXWVKf#Tfc(KZ;@a&bdDj5o{s_Xk_Orm z_Z4fOIP#vA|A4nYg0x2{y6;?03;Fxj{43W{1kJgxuP#up0aJ48B%gCEiU6h2ofSh96w^qKBkwHvZi_c(y@C1zlAN}E1u>HdhUC?jbpO6U;qCa-C^f|A zFNKF8E$V7T+U*PMqgp;EH+?p}?Cga(H=mH&w?+se(iJab4zd5{_mE=$*R<{TUAQeW zfwk#vM6HG^wAk7Yrd0+BRvKR+7mIXpUW6gGi^~XCPdI_A9)6+DHQMF-Q!mr}D<|;a z*q1ckqL<9<_dwUi6q@omo!QSB^Zz#rK7Nqq*;5@A{omK4t#u@{2G!8+H@th!u$FAw zI2jb9M`8JC5t@1CFMXe74KMya<>xYSVb zmrS0ylvLag0o&b+h95_&iM!)v)+&D>pp zN46HyDdhu<-V#|F;kguxGM5MkmK~#fN(!ES=wzd}9YWt7*+lunWBPkb0NG+3jCKdw zNX4mNAQKqR^IjRIsiz&zan&?&A&UkYEO}~Ui_`ZVM3ZOgfF9$x7pE?Ode(eu)i4uQ zthqwp-V4T%oZa}`Wf6D#b3F-NGeA@HTyXit32^$CIJYqF5gJuhk;!~+e%#1b+PrKZ zd@_?{A1p0|y|vT0obWj0_zpqT)g|~l&m4O4he+ku_X3Z)S6CB~jn?Tii7+D#)B<-9 z*?q&1zG^!WiA;i=V>zV##d`9Qb0yUh*Ko|x9$3it25!xc7WSl^g${nEpkgxyQ$nKP z!%|)THw!W49tGL3PF8j!zt>jwz~yi2(c1JW8Jl3v9Vy)_>`0XqK03TisQ7g^b}x9s z&T-oSpIk=^jkAhzoKps7Z#+#_e3ukH)^dRRE+)dtm?ScD=Ve?RGXp+F=h2?^HD#lR zEbz^S99UwqA1CiFrxnMaQG9ZPgv>L;Z6oJtK<_DBt-BPbJrp9>u8A93E|hifxLaj?~m)}I!~1Lte$z7a7#cLSJpEtz?@ zE`ixQ`6>Hu);DVCv6g?Q>ciVDb!3O}F*s?|#B;0W!2N_|nE5okqJr=BORH~#hkQ0| z`Dhay+6nNFXHY&g^99qJn()Bk3^*npV>jJ1f+w{`T&zSfcSQbJ`Ph+Jki7UTjlAXn z&npgN&Uyv||9eModngH=+73dUTsc`YwiLAXny}mL7IGuEYv7w)3jJn08RuSXrtza) zxgF+wx7q>Wf>A7#?*oL-sLK05ZLL@suzTQjDZIRjqEt{4S4;RtWf0ON+?{shs@Y?05v(_O0RtYsaG>Vuj~ey zn)n!voPQF(5NG;6>?Q4wUO)zYcVa+zAl8i@g{GN5Sb_ThZZZ!{M7f7Ea9wnnO>Get2RusnUU$hQaoYue%&R?;(c@i!2 zohfukk)Z)!gUPha6!_0E0&eDek>{_q$W1z$o}03QyL?~)t!W2#*P|+;H*p>eJs69j zE-r9rVI@BM?ZNbgj|hIW>VmV56%lnAFPv1aLvn+8CwJjAToCnYq;>v}@ z^TJg$6pz3IhvdQ8TA!3Oo*-7S5@@9yjZ+s&G3!gmqs8{MY~IJwLRr-+wxBzm+2bdwzpWTX-U1r_1-0A0Y)Qh{e^52;@qids+%zQYA%)9w_wz2YMVx5=YRSSrZ$ zFy!ZVHSR;LJ2a#pMUBWEbmv!Rwm&bBv1=>DO)Y=PHshW6!f+-t^GiIObkIWc=5ZKX zHkVFJXlF7f{)1%_V}v1GJRa4mB&UYPlkG|d$TgW@alm<4Im?2+JgEUAk}cFC{2{UO zI)u7=H2ADRAD5 z4#M;wUrF4RZkQI=f<9+*@lc%ul$V}{ajgrXA!I7$mS3?9;b%|#@8_}-h8bYGLY!oM zyF>P8->z70wSZaNpo14fb)j#PA^tULr8BjIa3E@!EaNjeic{!f%|K;9bp1hF6E&lO5%b)-gj(IRo z>iDzjfD%_)9Zf#WvL?STXrpj}300AL%{>1i!G@`&;=`Fo;7a-tI?;4B-5$xaW;_<- zg1kZOHnilqz*{PGruVQ)@VxweV*0(s}UBKQfc!CJMEZpf9undPIf{TC z8;Wb;rt@{W>pSmizwJan8%x1I5=Iv`QSWk zsytM&d?JI)_E383m@bk2eT-)d>Il0RtRs(;2kGYGDI_pSi~aU~3H6yimZtFeFVFlx zGI!ac>n1*;kGaX?xA)ZYtUW5 zq;$tRzo*=`kl?(HcJ)0QC}QBqX-R~vEg z;fS()0dAP7glBbEHusw#(T6}~3e)AlCI&c_H+D+%q z9o^5ax@-ufCLf-kGKR-lx8S&HGrIgeLhebwBKqb0cPUwvz2T@Le71B7joO-xv!;lE z&%33_rW(PO*?WK;lECFMpUCds7K~WL&^iwX>U*IN$(Cf$NHjrViW#4I-^V-bTIl{5 zB?yQZ#oc>+jK+lMGIRd*V@b>;m_IEK^4Qwjng^sydN#U> zI$K`5(MP|Xj%0Imbnw^qUb2&ur~YqMG4_@R#^?ppu>W##`6qek%zH_6lGLFuuHc6$nyV`xFv(0 zxcS3dS}v-G1~rb*bWww6yR-=|UAanbA2|x|i4|_&-hp^6o;V$QOr>J3qtEOxGNU4! z-u-ofu86!qpUbDi+@=`Z{U!sp)=H7v{9QJ9SQ%B4oA9*RX{KjqF^;-11zN-NQUAvx z&j$yEIJz3QMktu#EKxr#$!JHHK%B-gKGaq)%Vv00BP*a61MDpGw zW^DCV;`%KSX@e*+UfKX^=Ecl--IxlWs4O}Q%OQ85o|!vSNY$RM1E1}4G384jt_=i~ zov;Y!Xm|5Iv6C<|beb+~b%0HOztYBNf&m#%(N3WcmHWEs$qLFoS*0Pg&3 zGVVnIm3&x_ciW#b-s4V!XT2FJeE$e*Pd-M2Y(lSp$!8m%@wpt~RXo4i9xHE$&~Inm zX?X2n)>q{g#0*(sf6``hWLyIC=iDxg$~p}n>)Wt8Z#~|YRb<-lTET%S`52d*N1~W) zIK{iaJNP``BzH^jQ?OzDlV-z;!#&WZe2kvHWPzJSM7fesIVMhp3bxJe0_nH=(Ov8W z>=1b1mwq99Uh4yL&!WXjgS-2Q zm|#A+KJGMG9V{snmwUpxmbhUURx#p&vCw3xM>ac^5+{9WDAL@7BWwR*NAGMjOPqkZ z9}1a%D{-zjX*XOMM0QeREu%i>2A$LXoEW`IgqqJ6(fQZ~9CJ4oP55m0YQIjZYjBI? zp0dW=jw5_O=MbG1*H64m5dO;kgFow?@X7je@MsAl*R8zSe?cst@s*|izsy0`dm8>H zwt;Tk#4^8>l&Ddno5e4GQFw7t7XQi^;d!~!blus_q|N#xwe@02;)zR;=iP!@WBq8W zRToWp%Ab8^)sm1`I>cgcGLa9;qfsNb&}pkCY*NcYO{tT_(c~ts(r=(@b<$WRF&0*R z=ef>fPw+1ID@?w~OR6XNg{__<0k*EK^m%hLn=hJXRUbhO>5C^*P+%~=o*+v&!KFr)dU5A022dMkD8`SbaKUs1*3>y@KQ8?a% z_z%A%!4mD@s3wm?p1Z(o;02Q}s(^Ciw=v&SW#Ok9;z%LT3$`3-&8(qYn(Ty|^ETuC zs5_)#!e!9$9bsYW7%E!322IKq(+eFLc;HkGyE^Cqv+LIYv+A@2u6Kz=#!ZSyZfJs$ z=K;*+KvODI{=r7ZtN?`tONddT5%Jui!pt8Zg;we>$=FO|R^MSI-b|c><=<7Mq_wg75!tQ1Eo}sPNHpsEf#xesi>16U2&zL^t+Hu z|L8_P^%T?jYf9Jy&_Wk6X7J~}62FI;K!cJL$?r@tk`VZU>FfT^Y;xkL>&XgQKPQ*O zULm+<-w%OinmBnXG{QIeOKH8^M8@jwTe^D509`mH7k{5R0&Vr9glV-M;PtZ+|BHCT zaM6S+MNVOrf0~2-n(x@uLo3>$XO%xy@!54_I88lKboEVw{7h0$g)5sL2}fFLS%G#oChffdjj!YB1TSrDpMH_<9_vPx z9fVxnj5Z=Z=tMkLnsei{cxJ2UI4)Mblcub#La9c6-z)EfP3N>BWyvD4|LsIr_Ggyh z&vyaHWX+{x?X75n@m9WrDhdW6TUk$YBm7)78ojkrafhS{>ON~_M%(|SBSVFxYJM}! zQPsoYY7z3;p`5z6i@^7(njq)Ed#KMkkfxvOX=zdcF>%`p#%9t?UAQrM#EqfXLnmPM zA~UGJ`+&|B^7-H0weVB(7Y!-WU^Okq(w@`V;HeW1d6x;=y;Oxyj2!;>^oyKvS%y9H zXVbkOx5HhVv0!uMHgifN0>=bamOJn6z^sysq$YX^jrK~zkL)60TjfNq*o@wLD&zI@hH!{?H)IWT*?h@*zZKg|Je4?J0ol$Nb?@_6r%lEf^@J?YaZX6p7 zBZUQQXnYwRtx`reoNB@||3>4iTzkCr!v>YD+sL+9-e)vdgzi#&fFgCvQ2VL^CyyH- zacm7e{;G|PJP@Ph{b|g4z7u_X?gLbF5Et6g!_eunfw57X2=DH+QhIk!AeMISp+3Cee&ND%{v=L#(lml|dYoTjPu`$-dL zPOi7B3M&kCIBCNuoc+v_`|Vl`|NZ9#MeTe>Y@8n&b`hNQOA6g3c46jJagdDQea0VW z2v6-Sq``w4)VkOmY3y7`@6f}AU)C}TkurGsloNb%(!y2el%O;AD|3`*K`_=LGV(i{hF9HN`wM`ByZ3@W-i0!`(HscN}U za8uHs`1I(&FTEuCoM)`2iNujn();1pk`h*P{5)c}XanQ;hG#3}#M1?$RpjBZM)K~? zBKkvfh?IAZ6NcvUz0ZX*M#78|1kp zjgM#?Cki!(>&b(m_mtJhg*QCkTX8`gxH&(8!H^+Pbe@J!uDCGvm-3-pc_Lm4UVvkd z+#&rl4TS+FuW;eTbr2RLi+awZu=A}6PRM#hK2LXLIwY<^hVBpYJ@_ZJdHEhC>z-p~ zh6YwVv%#Q!HFWfF0&{CH1!j6Lq53;A$=B96;+1)Th=oQ#H|`*(WP%ao-a<#?GaO&E zjT}E#4O@@%D~wIxJv53@Q4snp z2|p~DQ)V-nED1&Z38)cy2xyrTT{EWVDc>Zvns> zI>0x#9;%mKiYd8Hc=+jiwsfux?hYP{trJV>F@=*ApQYq5QoVy*K37LhRa^w`8|EZ^ zmK(RYtcQeORN!8UekH4{Z;_THZ8%_`!#hoc^unkda$uDjr{LCty{gUNx0ml3uUL(y zx|cwlXDi++8xIZl+sK3J<@mca49a(DvO4NfWSlUAv5V`%Bl26}+RH)~L_af~IuCJN zZUB7tSP2VU&tYkSj^$b(U0AoakFK8+K^HWN(akki*{swlJWsL(9)u_1q>WPvb1NMB zj-@l-S8l@hF{va(TnLjVjKy`)HrPNOqWSNw*q`ZyJ0`_rcV-rDcu>sj>K)IOEf6yQ zZI6Zn#lv*_@Jz17Gm06rdKX>Icgz3i4$;!OQW{l-73;U;q4GT`TDQ&<`vk8TO)Cvx zt;DHVj2~^;D+0T-B;fF>0@g165RNt6Mb}AwWP2w4K#gp7+G8>xaJYYo9-i`nbuBZ2 z%TWj6_s2BYoBxn5e=1I#H@u+tzr>Wt1&Da?y{7{?Lbry?@CH<1uiexE0I(tc8l#9Np~{1^dy#~P+c=Llsiv|POZMPylXV0%!jZ8oWV0?Gz)3cTL1$_c zsUMbsa6@Mz-CxYqZMsj&zW0*_-7o0!$5M2yR6R(@hJwjgTdMDIjZPesg2(LAsp2Y2 zuzMNBv=q*RHM(B`svJno^>l`bN`zG^(_u<=FjYu9N0J47L^LQ59eSl;t#A?ZJ|-G| zk0^0_ww)#4r2a8>F%A^2q{!shCIqZ02Lsqow(bTcv)3SM5oTQPT8eVuIf z^P}z5jnBH~Z1_nG`^@pM(lr`BSqX&O>#?eDHfGwKpdB-L?_dQdIGr>?)d%@KV;!HN zDUJb=z(5>)WkQWEYz6zGDHXfLPT=gia>i7$gIq;~q4j4Nn6;0NzqJ^H1Kcrl-)d9{ z(#Iz)qQZiw$5Hv64~pI(*xecd7Hk;Ncl9Oq=LK->b|+(%sLmG(Kw=}$|ksJ5lw?)GPv^`xE;8O(`r^?XIlwdCh-hUr#G`fL08RX zuV|6dZe>~}BLi8z_U!RWGu-`ITzI+6n7sI;iSx3T5|4eJq&v-kh?!X9@xKu;VVM&a z9XtS|b`;>1DZAj^AAw*}xVTVJDVwp*{6qKk@_m2{Lhj7%WAsDSEK*b3K;6zBqLJFU zMEG+e5P#S5)cvA3l5QmEXr2LuFVdNbZv~h>XE&yPGlyuW#W>AQ6Z%~D!?!*DpnB^S z7C40Cm_>KdxXBlV-&}-o6Q!_AS|4VpuOsXJ1>m8p3&}O@+2G-zE41J}&^oc#uzut& z>@1x}!hFAjWa-+9!B`Etcyb!7lv<3A%cQ8Qd==~~(gv5Jy|{q)Cp@yFg2Kf~jHsS9 z8Z64jeZg~KVe?bio)!;Zq(8uP>u@;y`aArtmd4H1yJ_tyX<@^+IJ_`u1!L00IisUn zu|i7%+Rol3W8$Ld+WCv&=}~zaD=xsvf=d`<9mO8r8pYuWW$w$D5;8ny8qN($!@ZJO zWKvcVk$8QPD(@c#i-HXp+B*hEJ4nDIxC{y@&+zBvNIZ~i2~{qWu&}WX8}wUo@KY@u z{?d;J*JZ-9ZStV7>ouN;QXu)B>g4HCCCiEP&r!2516Ze_3yqh}1rXJ3lQ z!*^%FZrL0RKGH+==cR*|{1q~?vk13}OXIDXDP)G(TS0USOMZ&TgRDa~?6NyTkF3c+ zTmM>E!vvs@fgv@>-NX3Yj>J5LDWLo14vAZh( z@^skOQAsmRwb5}ui>}2YOyS9G%;}~?EU)2vmfK`4wKf0IFJ%i+hx^HUk?xR|N7?-M zs*0w&Od==Cv^h0RStz-$i^I|mf*lg@W-t6*5X8AqlyO=t&SfrV@WLTkF`%!d` zR-k8RnN!pBYIY;#T{83S$jMF4boz1ry>w(5T20p@+CB<6^m8h4Kh#X`B(H;^o38{P z_n6_v$rX^g^AfT1HNzaxE!SVjGXkf^p+>GQ8)*NAF^&CH+PTPpN_>~b_{^2Cs%H^C z&Q4<{Xm;2NeVsL_!G%ovAS(weqW_hjvJ?jir806< z{2ff*Ap(t(4pb_Fp$d}MU}pI?95q^#bBTzh?k8m6xu+pD`8uDP1%%_klL%7n!h757 z%OUlTC1<$xKX@ThL6??!vsbT1!wvI6#%t0Awk%*49-9x@YY9$R7z>-@wXs&s9j5$F zq!#?ni*!b!Lt8e_<&h(mkL|H*`&qbi(gi%{PJrgB1iafNhb=F(U`qBp+-JhFp@AFT1 zp4Kg>@s5F9xpVA6MIZX%fFGT**Miu93N#Enf!*_yVOK!`WPU5534weIia+DVEh>Q{ z$DQ!Fd-u&3_AWsL!AhIZodeR+a!>$Ve826&+Dl3 z=H)!Eq5;FgTcBPs4mWsR6&!w%%5%zBq3&qDyT$0UGxvt#@ri#>^i()Lv6bO0aysyt z`cw#7brdfD9Zkfh3dr*K9P-B}1l(S$2;U4bY}>OBZ1}WPdQ3Ws4OtmU9FFFZFL~*V z@P&KD)UbYH>1xPzTA$@zbmdU_SP>JGYRJFOYr(B&CcKkf&ZW%0iYnSWAv4UE?wKVI z@Y$T+xWvz$^V-P-u}qTh>rYynpHU-l?Lg3iea2%OC3b$$Uy>&i=VL9m{7E0V;4c|SY6CW+0U;n#8Ni} zAbZyh5A7Bzr!@ z;@%z&;SYUXeEIxJ5Hkgx76W^oiUCXF9scf65tvgD7>mu%SK-@;F<%+!;Fy=%qo3#s4QDX zsR7@OSf)$5PZiUN>H=Jy=LL}y3MxKoeqi6u;B!4&y_u{0=e}iZ35ukaqpr*<-ZfMJ zy6GyAnfsgAzo;Z9PpAmTJc-8h20B7xaR;z{y8x0FYEY?N9T?{>PKUjBgQ}|iTG&j{I8N(Uf_X)^(|x`oXi;{Z)EsKT(9}%Y`%s_AmC(*rh6wRG+0Tz6%W@GlpF)z37qn~ao(AuPYc)hZL zJ-<4HxZ0^ucD)a^sYkUAc zT^Ylsyq1JL>D9#aRvOP@cY!FeT-Lqc5Y=uR$ESQJZM0Pk9Vf7%!|o2$Vf$BN_+=-y z#f-*2sjILoSDR>*%F^DrU(~bg9}|^ZBG_U%jNV-ybh3Ug#{10?9-1mmvW2%Xl5MeoPO99pwUJpV0;Sy`=Hjc|4f>1hZ%NkW52$m|Gi&@N7413=zOW zhf1==JPoO)GpO;|p|Bw%?sU&9*dmbvANStDhnqyuV8R=^FZc}Uhx2IqY5=BqFW>}6 zjfKj&o*4d*KR>Fu@yrMfE-=vxSI0dPf#-ESbqXtUQWIL%!&HSQd%fepXN*PDZ*iX`Xu@7&!ZriLRF5 z_CgiDe2{^-fur>NXLUYjo%wO{%fP{+;Dt*_dG0)AMyVQfCZ$r_h7EKJ-=EW4ssujM<4|SqSok4~!N>Ou z0MFQx=MN5I%gBAU4Wzi*dp{xAiiPbXLb5i%4Dwr!(i^r$MAhF4lSNX&`{^!-?{yWX zK24>0JaeM;jwznJ7>UAjlj$23B^uwt(WCprIfI$U@n^n|@S@ji+MrT{6~(%sC!sAS>K;xaQBTA#hA|GDvbVBM91{Mju6*^W30ee(o2ZaunX*}Pheh~_xtKrZ7bI>|rOu*A=g!F{aVzf; zKY<+{GrU2U1s!M8uIYf$TPvo=P!hX>j$?ORBVBOfe;7IsM=IMejN6fl8o$-knleDK}xA4lD1MZ(o$)vzV|P{ImdgR=f1D& z_X{W0x7T8$!G0Q?WeHAm^GQ`{2J|g_LS_kHB^Rbs(th0>cExu>@Xb6l<$BchYu*!A zc{%pv-)7qO<0)AG6ood{fjDN(;?spCkXELE%}<4Jhp^uwGNFJxGB|>QePQI**C@^_ z=#69Re^SQ}x8cV6%alo0!2A2Z)0_@7!Nk?YkeT{{L>)=yIc-0Hqx&ZkagKH4wLT1m z?(`8q7Zc)hBM^?&I^ZpQh)!ntV3HV0i@#`NSWPq?t2_-;Whd~5E03@rIHy2*Qz*MH zhyj-;EkyX3jNrs_j%%~M46PVNzVpkMXkOldgGrj0vdkC_2JaG|`}qLc0d#}fEf6~_ z%wGMkyxwD#E|y6r;`9YPR0yf&W{bHvboc^$Dz6=+gyzu7t0t_C_*4>Z_YQ6!h^DJX zdwGkMSyrZf8Mr<7$Mm>R7|cJ84x3h*#TBoCZ{gu^dbc>#22#3WrW+o3KA&W9dAUn2 zf9SqLYrr?+2e(viA{Y_~4Ar?wQLF-pq(o6UjnP$>%R&1=zUc0B2>Z^v)9 z)u?yb26&fPk0rakaqWgpSQUGo=g>6|E<1i^&(-IFZOBTJk*`jyIQOU!l?0tB<)n1^ zFkNkGMay6AWN+$J!4c&Mu$sifCHJ&2B*mZHOif{peow^1@tLG5BOd3A9RtsJDZF8D z4s<3=z<@mup{K5nTFHtD2F?V)`^tUzF!(04z2Y2IFW14mA{ku2f1?jl^ND^GQmwKw;a$gD!KRy$-(X^jkVNN%t5G$oCbn2rbke{L^=y;A4!%~s_$!G6sUZQ42uEgQ@ z7`L~m;ryJN!1awZQC|?n7I>C{P3uR}ldg!PT+XtgtCzh<=kskh`Jhy;2x?a>Bl8l^ z(OrkQ4xaF1!X{`7`t)V_`mcV_wJraVUs}=h;YwXRo$#0(5KYIq0TvuP>jE-u%EYiG z9=;6!qS;yn*ugn*K5KNtd7W7#kqsj@ZHuAezhyw~{Gvh!CZbMC97w9I#;rp)LFuCz zj{EQgol-TByYkSG^%qE=Ni3N)XF5cD7{yb|57)147Du_n@u*W)$cW^9p!ap0Xa~n# zv|3Y7cY3@>-{h=%>!7)e)Vuj)a{mM7*(D{ui%nO}1 z5h9GNiOybCOmWr`ERqpr_EmHct@S&|=!0I+yK@wSRsgxA`kJj(j-ro`U4wDfn*10d z#V=ky1vW01f&1q1ye+wV!IaCuhb8yWS#c8tt`nqCN~n;&coR);pX1ZdM*28c@ghC` zBm*Z$jnMBe7SgY->9D~40nHOy!%y?- zx;*a7bir4Xq|o3>E1UmSoS*-mWm0;Fm_qH9SpAa63~{bV-+iBmwYMUqIVwT)#f6w3 zT7@IudY~@eo+fuI!+pc?Oba|Em5oQC>x&dCxBeH=l>b}5(;%IZc2)(uW?vjJRDz$D zalmxX(&3G_!|B+4$U zXSR5}LamoMIE`O|Z5j8d>FP5Ot0f9se2mDsO^tX&`Zs+XbPks#uV9zojzpR1i%D7b zVieQcLT^;=CLda5u~%IJ{#1)n@qyELbdfSz$owLzDfxI&ItD-8Q~=@6S%O$^OIv%-z$;=2mrMCxu*KeY& zF>dhHS6|>ZMFL$HstU}?lm={{DKJW+oHt_?b*V0(!(HQWM(PQ+&3y@~KiLPBg5x;lss?}I+;gy3 zDwbV2?hd_{7>gSxZsG?ur{W8;hWc_@(H%iS0%PfS)PGMlzVWWAzwEM)?piBB`X<<$D|EM!PW?ReNsFfkUm73@RDt00+u*byjm}xS2E~SYIi|lMC>&<#;qnAhu&IK0 ze?3kL!s98sE0$GzBrE8gD-UMF)A5h`je1_iJ)FgDzavd;!Tygv8rf!3>!Au(xMm*z zMMfkEkc5NqlK=k!Gc7IEhxG5i=e z8|fo#1;L4f8+j+{l2PBLfa{$H?4`C zN4C+#v#!*5*nf0u8Hx(ws^~LRfC?|FIN#(2^0HzH$$6uWdncJ;>((b!?W=%{s=lTdpo={CTutb+ zd@3_bggsYujq^izfp=gY4Mt-jwP*h{%8;0er1LlYjbGshrRfr_*VVF4{u3{ zj~xmlEz_WA#~^;0L{%`*#NCPnXA< z&iN?sv7IrhK87M2V|g_yYp9MOpPD^bD7bZPJXyALJQ=r2Pq3sb16MY5G3#GC)t`Di z8+R!0B*(fxkX7k#QSQuKH1W8LF}o+guKI4AA!17G=>wcO+)oxq8)EeFMPPY85RGdz zVdVZ=sP%oy<%L(1x_8=ayF)xWODGWSzH-uhdNK;ecCgEH^&w%AFP@ZM| zQ8IlbnB4cW#FdM6iA7Ksbq*{dhYh(d4fsJv-EAoBJdXEFzpA4f7+jH3;}`mBurxWX+3W(X43Wmu>zdd~8zX$>)PcUe$623%$%6Ogg;agN zAs(x~$6jsg2F2HZt`rXbAj^Kuq)9ts@rQ;Y=bU_qRu>zv;%WdbFq5RyYc25UGhMts z%@8;2zf51A3Pm5|m0%Xp3a23o@_0?;>f9}~d9fCDJ}+g~9G$^9{}jfucL}(2W;DjQ z{===7?et)clE8+mwy(W%0j4EnGfQr8e!opu@ke$mn;f|tM>owO=IgiJ!qtM32x@5;n5xDu)=Z^zOK{YUv}S(#qrc{?Z{zN z{1w4l;5Ux=_6eg|p9R>h>LZ^%?!fL~G3@ThLD4il^b*}q%=L7zc;Q4rMywk45C5P^ zE*vA@!yj+{3?b*@I%r(zHc}-!m$iEFoflO11YVIy>~h@!LUAV%Go<+=rIQ3>izf-( zBn)}=4&|`BRFP&np2vvmz<&{P3~bLR3z8Jw~`oLi;398e=6S2pC^aCazJ( z$V20BMfDauan2RD_ovY_GWr;p8G$8U`tZhF5{Ey_z`+~maBZLkEW1-qEi`gznpPq% zyX8t{AAO^4B~t}=xGYKfpF!g370WK zb8XMj-Uq2DWd4iGo<2dv2aC{Bv5B3%&&o{%&$Ek%JJNgWhWB8R;v-RxnKjU!M!o4_kg)Z_m%mj7?c{KX`ak3@DgUjsp z@=j|T(cQj-y#8Pv*6vX$Rve!M6%$UdiP5p>wsAfYJEV%gcb|a0gKv2k{zRgwsR;V6 z@S%qDY{+NFE%fl7WqkMi)r@J{M|$nDB`sD{5EPiVFkYL+|*V|ud#QU-wFUP-%L5X)G-yc@B zc|B>|q(H4JxO3;fTntw#qjlMm_$Ew)iP5NI_7D-^c|T>emXD*#x8((Mj!DDGBReqe zOB)$}E`m?q{-Y74ciCH`im*VwmbrP_89E+rhV8eN1PZqwLujBc=PKEPyX}%`fZHze zEa@4mfBP7H(e{QKIbO%9=Ynbf;0Qk5eG6S*=D~Ar*QI<}fU|Eq!GwnMkf|Pu+P9uj zD^+oD_#8!UE1kfX3eKDd;~>trq#(~v+p|%yEPeCONn5f`!+DV9KmF0+^6PK^Qcl{ zpq{G!bnA~2Y?@`n_2mo5jIR-(kkCojK0vzlbRDf+od_zJO@GIx&^r~S=-1rEhAg*- zwU7I;e0Uj*Uf%{!^1hRK4;$zY)kx;xU@Fl#-bz>4rJ?jkOVoH1%Q{{CL|pSL$nnZ# z_*69o^*wY*w)X|LVYwYjrzAp^P6*v`#~7-!SHptm_9WRwj=%21b4KZKG|HctK)jE( zQICiqC|#flwe$V4BtI4M4p-B(5>uk-_ZU4|Hq&SErBr5V23~nCjl5Y=cs8m4mpnL2 z>Ssy{_80rY;AIDTe)nYlWa5Lv*E3zbGF$Iy=s)9@oow%-Rh00%V&`t(`DcjKZ6~%&w?ZI8!49gQjJ$7B)C6;uDQZF z!=}YR*uNe8qgDyvF)tdGm<)Phg$|XBYC-b>3AC9uNKD@OQS+pg@V-w4({2fHL{}Rh zZA&L}B&7K(Ki(trtBv{dPVU9(BO-8N?RB!bVIAq$OUK*&m$`dmG<);>B!0Cx4}vMR6J0k`-pM#9qm!NQGAl_U&Or51!G~Ldz zCl(Hqd!F~nK=@9G&T)YbPJWYYEy9=CuotJ8)YR)edrPe}3#r{dRg617%rqzkb`Ep+gdt41@rrt- zt|GG!K4OX`#^~y`kJ$8^Rd92&A3b$b1vz~tsO`+bQrjn_`G_?YUwoEanWaFK&+UQX zCQX60eV)J}VJgN7OGC$g9WG8g8-`3D(KXZM(Es@iyl~|yTyZ}R4{VpQT~5znsaZX- z^<0Aibu+*)vzXd7PA67AwwQmh2Icz8&__;z{>sZot3w)$gu!AAC~}}K)=!u-79kM$ zn46i;(ZfCc@|3G>fZ=UnRLXfxhmX2(-cc8x<%eVJ-Jk$W`)Q10)s{G0c@HiqnnK4Z z@1;r(-L$;SfVDLohRfG02|Kg_+@)h_c8(Ub{ge?r`zb?PM|0WPkG>F}#TS|DzV7sa zYc72Ltx5yq)WENZ>)Ff`=8LlB@Po+X9q;KltH21^YzZifD?_sjYVe?|j22X~R>?z;RI<10kDQxb#X?_;XpPTFQO z5BiR4lUgftSQci2p;ZmUhx<-8e>;;#bOGHW7R;OuxB%;|6$Dqk?6`b$B`PHwX}kf6$gIHg6*)9zsG7+v z{eZJ39fNd^i!qe_j_3_Fp!PBc+AC?q*Nr?)VyC@^3eR0wbk_znv^52xCp77y?rHEs zBZxd5>@s^TvlE634$=PEGGzQs#GNn8Q0aCWj`?Tf)0coZ z)I(w56VesUImry~v!{-ogZoJxBr(+(2Y|@6V9&c(B-Tch<(Q28!2d46mB-)6+j<_o>SKz_%jc4wO?lWr)9Kd0 z1m@jMWBOpTJbcRa#3fm|To(5^8qev%rZxe|zP1SFev<{A9C`3MGX)n;F2v83qIlRV zk?hfthOcAOVCJ4wc7^X^_DTO=db{}(@zyY)RB}DB2z*3oc{(=vq|rw~{&am>18Kq^hhM7Iwj-Su|v-AM>Cgs?E432yVSGy{PL-7vILf%E#(>+~dc_{(H|#N|2gIN=yOLk-arzq1BGJ3;faD^AjGBCooqWARow%#zta z+EywGCT`9p(gQ85E%(p(Z|oTTF|!QY_)qB8z5?cV=m^u&mW;mDMjXm<2KM~(qT|1% zqe5*S!i5d^X4eE zZ*T_vpy`CUZVn(mI*aGxri>py*weXBoM_#%RvI@LLX?lS&~=A&P*QXbioOki&C#iF zjR~RXz&SR88%gfF9Bw~wh8)LqOem~lw2#T-1I`!a$9-o=26e!IOc%{RGzZ3iTYyu- zQrXN;wIJd2mQ?)8XRd0!qVLSYA&n^0Ln;$kGudM}!TuBr6Kb*TjRFK~JE7a+6yiKO z$`mC9V53qgluv9Xxep8A!`xXc*#E$8BNc4bHbu)nGTeSKpJ040#cTS4B$`LBOux=9 z4N`(bsg|5az8sY=rGs}%2XE~!c^EItHM4ftvNHd+(FprmOyfGF2Ns0j-feeSvt?XA zZ+|Ua+q4|k>&4=z$Wsg%(4e0)3eg)auwdC@I=w(ta6a`9U4Kp*9oL4Ew^l~fO>+WB z$qb@6vku+gE`*%M7w+v>Ek$SvZy?p*>PG14#-_2 z|6Q3(+$U_r3HN6)PLq}K`NGd6ZPg6MJ$;N+M)K&+7uBS5bPFz>qXoa6gJ>77hUZ6e zp!-1`k<^@q%g$yqxAo+xmzyEas9`5sy#EKB_Y1CZtm8kP;l%z*HJ;PRFq4=PjgPCV znD?1a$f@*JGp|^7sU|arYY2MV<{^Jb?n^dqhN85icy9 z5Bm!*l1Nu2SRXhC;#(BRU!i}H`^A|qvv`P)D#gh;lY=-TvIbRx=HkD5yD&oNJ{}Mm zrGNiyqu!AhSmAle?50zu0?!ZwEUn?*GmUV(sXhx=n|&baKh|;`_*SYZR!T(ETUn=r z`>+DQ&pFJxMVr38(($8qJDl)yWu zg$^H_NF(=!GJ_|6lfRpD$Q(f;=&9?&_2^Md)#N(&8Sm(G=k-LiWd^xi%k2(|_mi8C zYoU(oSBA%j(y8tu0^fHLkm>T3f!|B(Rhi~D1sT;aoDR} zPw1<2LWPQPM*EeFfL4qI4Xy z*UXh;jaOkuy8`SkI?ZL^9I)e%H(U%|OPB{Cc!f@ZvVdN!ThR?Y89V8B6D{Z{w86-s zsgQ6p9tWi~AV7X0I2cWVRX27ZZ+b~xknJ=)J~58^PtB(v$E`A(a4im(eiDXGduzIP z&0T!3LK4n-o}fy;U+`>f+XD%5tMmF66f<_$XayMz_*8gnd!p zbT9m9iiG3nL1TsPGR@qM?P6;ed*`tbIk_sBO1=yryvny!_WKIro9xV+Ki7x0InJgF zCoyog?h{SwjHEJ$TgZ1g3wZdWrGC~3LCM+QS&xiB90+Y^6mO~EC)->SzAb@Vu4I{p z0Uk;3F(S8gT=3pFLT?qVLJM67M*PBfnjjDUUtY+BG)s|@y&_<0*-391 zXyax688THOlkVASN&bnwXZ(E^(UH}|WWX^WdY?+dzU6^5eSHM@9D7U0dJj^vSDCT9 z6N;1CAL6gNB)m^f;D~lF`Id8y$~b$F1&@WmR<9TPLS?YwO&-a&R7H__@!X6@5kHto z350&+G9SOKfal9w>7Ol^=%mgIo;vLk#C z5y5VijWmSI&9;|Tn6=iRC#h@C^t3dipQ|Uj8*};h(-S#Pl7Q|T&vl9J zM6(mD1R#<5mDY&GaCe6&qUxIkj#_PG+hTqCYlSFZcIhTuHno(JX3iB!lgNx*O|+9A zBr7H!hNY7yk~y87FeEvJ|4cF+%WJk1d9w|)Z~88*opcgEsGh_((N<(4=gj%)7=slh z7x8A7w7?_H0T%u}03Ll~#Jq&hi_|DVN5?j@D336vSAJu>s}c@Ns^T=2X68xLTttT& zqBuC7^7jbyZfD(qtd!~e;l_Hhx$QJq-t|Glv9KCLY`$ zixV`P=;TLUbY<@sB5v=E@@;AOou*=g%m5oQlufRgMbncen=n92m(Dx&1rKa~ixK|{ z&~Z`}dTrf7Lp~+rr>{eBBWsx6na-VU@0tJ&mJs-V+DEk{bTQKIC{9YsL}q;pl^d)3XF>_O*sOb1ccRhA@2YBY}aJt7#Xxf-xJTN&KEC z^pfrZdheYPMj0qzQqK~UTe_0!uhGNBE^1WYXg7VbsGT%jZlmq_&iJYcu-b@YIEiP| z>*Kili}EwLqx78~IwTFFQ@$|GZv(LSzb@hvHyzETHE?7mj~;rx12oShkm**%D86uz zxQAWAEC(Sd|MG=i?KZ~GJEzl?K9%f1xE8)%=?8HN&Fm5N86-(0hxXX~qK2l$^%)8i zz}NO3&dHgL0q4H6BimA_Uvmz5v^yG;t$krnlp*!0pS&nLE0lb2P-0(9O(Xq@qpZZ* z38YwjGP%Pn-D{i;!TBa%$_WL%yOZqy!j1G8MlUNf? zDso~Y^J+CE?tiA!ioR9U^PL3Ud-iy}A=e@H{Z$Mf?i)a+^lQ3jZ#mn*TL;gk8lb7T zE@NTqN4MI=qt&0OR6)fLeQvI2Z*-V&o{$-MK;IQh&n{%&ZZ}4vm&V>&dk@1de>dH5 zAK_fu0@70ziCI?^;9K52P}aS{eb$HBlZQ%h_6-?u7&!@R&p1F&QW4k1*hddEyrkbI z4iMFc=XnA38sz;~N^M&Hl0go)eE&)(opq=cqmCqEe5)5qKN2PtMN#m0bsCDA1;Ar> z1;+fqCc$S5BRUY(iSkmls9MOe=qHZS3L6nYxNt4K`H|yVFY%>W+-LX9I4dCLeAwA- zkD0Om5nk>{on)yqK6fZI9WvewbJ&G&_>?*uke@))v+dba2_wt~iMeoSK#i@MUrJra zx=2^W3V3~TC3*W;8%DC%K>f)$qP6EbDViKdR6M6)oWndc{M$*-y8b2)-*SvI+<|V( zVrh`ZUEWsd3+&Bf!Z>&695qO6r^mm!Q5|LmF7Oqi%XCErSBkChUZ54L+!_N~@$tCj zSpxISa0RC7oab_w9^m<04l5K2F#{QGwDnykaH@a0k*{iDWu>bk<@ zp>S3v)d%Ojo&^4NEU7a-O3X||1(Q#4nQOU6B>VuNia{=xKf8!No;J9;{T#+-=wXst zJSnmJ$8KGm#Qw34fq5+^Sn9Wu-T%ZI`?51pPkA9-mGlGqcg)2O?(A`n#B0zLdSlqfbCgY}^*{K;H~^q*2_Q^q-$Yomy%SRK4Kxj=+{453^? z0Vi~OgF(CuYSqNDUUn_SCO8U(=7po{(s0Hwyd5XE?BYkS&89lTkLjC|WX{)T2CB;K z%-?AVaHX~zMkOzyW=B7nq2EH>wbPg~9Vz-&kVy+eA96jj^K^3Y3H&B*PP4P3@bkNy z#CNF-%;54Pgn06jvJmWsFO$Ecz2tV}#L z`+iCo|N51}hORiWIf|Pr7k{Pm4xFQrVvXpU9>+e^7RPf%LB#2x1|E8No~$`?obeU5 zHEX{6g=WwDMlkdZ&ElOQReKfCY0*S#yq%#MY9`oMUxFzd_h(klQv7xC92ROsGOmN% zPDN=YD@e7(veNk&*l5LZS9{5YWOLNiY~+|*!q~L1k9qNYJQP(c;jtGlY4g|rsN6+k z;>&EH|LzSF4<8e36`RN1TT2)z{j*rQ_6pf(!eVBh+n2<)zui9hGR#oyfE|a}z?ScW8VT_B2IlUn@l`kCGgD&d}$?RfHTy2>| zw$4l^^2$1Ru}%n@=KN(XvP1uy0Z#7w*V^&uj1u@cB7qyf z?&o$tmDF8*vcO>NNgP-b&D2=yq2paIw7EdZ{+$|-mKa8N7$)+zJ$GOtr1!zZzr{qc zp_pWzUr+4s$rDS?;k!@u8vFU`E_!WbA)9}?jhXEJi2dSz3hwx^birOvy5A}n$KSoi zeHLf32d$1_OYJVAupx)m$zEhGzp)@|KTG2*<_Jhmuq2OfFClsXW;Ab73f>>K;q#Ar z;Nj4fOvjsOcz$>ZW?KBgUP-j4s+ZK8{qqmOf;yg&q)R6c!lMHC%&EQN6nCo#*5 zbAANM2v+1+(#b92X&Nb9;wsG8L@M^h7eLvgbQJVttWX@g61Qgc)m53nS zZwClxJ;UN7FPW(YQ8;(=9p2q*+i*haTqw>L#?&KgaN4a>a(s^{_F1jro@WtwE%pt4 z+LlLxcAuw1Y86EI<$nCTD-|WzZUF!4TsSM=M4Xaufc@(*n$$Rf#u@&`qWUQI)SgVD z@MDN%=e#4rzK*QWw@O}0dkvX>a0S?%a)BI`>4MVJn`zqHMcn?-00(;QP=w3!rfQ#t z7}FoHX0s){Sk=qBpDBe+KW|gZ{i$&Hd>2)DAVP!H9Kd3_3K}&X<$h)uz7YEl*9mjy zJ1vR>rhG`>>yA$?jFlzD*KppcGrShPCR)k$e0S!I(fs|_nS&Pj zbdK_2lvjCsN>3M-GfMCbDxD$l-;$X<*;lUqA2m z1)N{O((vUqG*;&>Rh~2j3p-<&QrisZ|F9FQy_Ip9*LU)=MvSU_`%9-M*U}%mR59x1 zct~uMgPhm9*dWDj5~t)zPQIDRZ0uV@TsXdzX8&SLAK66a>X*<57vFE!GBXAyydf;@eqnul#$ zfK&*7ry_mtsiTu5#OKy9vyE@UnL~+mVQDnG;rk1=AvF|6t{!W8>76SPd#*y z5r)yUne5TdU-(@~Sa9)7EqV8~1OA)e#vHdqyr-&w`g4-dzvvzj?`go8mzuDdV_x*B~UL%O#rls9?&M*2Np2Nv3bY|jq1VD2Rv9B#{W`L4#*>#pO>H(}(Ddnqnfj%E!S z!|3~hGrVaxACjsfRdkm>rWDgA8N#< z=?$GJXolYd#%N{1?dncj;e%Vks8=nGD_dn~91L* zN-et~xrPa9j-@9%i`mUV0_KQSHY$BA!%my4nEg^)Fy*%;I&HTlqPI_A;?#>YiSr(t zdqseCRweUGw2e7cv=7tA`_j{ID^WY0GEQS^=*Dv<8|Gzmj>uKmQmI4TSAS%`uHHZ* zSI;E-0{pS`)ImllH5uGgZsW#~cJib6GRA)Aey8FvvL*CAEUjA%J^snGdY}|-L$k@l zcPfHUA8hc`12vePoKCK-Rzi!KEE@j!I4TTjVCnQ5cuD;O+|)WohRJMDnc_|gZmF?G z>~VCtsvx+ny$srahLM)m0t`u<#y4C(jybmbA{m_~O3DkY(6vbv?Uj|-1(`|oQq(ol z#<>s=Y9Aq4vL%?6tpf^=-x0sDWn}Fs3Be%~X@LRlr+1d_MU@w8`BB#sX#0{8GGjp$ z-i?@#*IUC$-y$AVSfs+vp0$jr-G8{G{WPfk)5m};N&f7m1w`rYR?OLG2w%Azx^~|Z z?io2m(jFP0#Bw)&h23r1tG1u6Pv4Af-z?yj+G32Ctl=`T*XhnqH!zy#fjbV(K=Cst zpt`q;$xjZ1Z{>O9b@^(NpArGyr)7zt;tP(wYay3aztWetRM;(x4x*3EGP;=KVf_ww zB5{qJ$7QFQ;9Q9~NJ>q`)^1t2>newLTKZ{C+z@>%&e9K3<)kHf3VP3QK&LmK;4L>_ z{T19nC6vRMySK+dOV(;C^4|}7=tnuev5kRi-iLUKvyDkuV=_!>R2AgewKCjECcXjSNG3W{^{b}fYa)6zhnr^ zm|+5w8|T2m{RK3=N1mQNQGiMlL-C3>fywJ`(12I^^wLiyzWEtlToW3L@wLy%GdWc( zIQ4;g;2B)CU7n^qVTrYt0UB6d#okeIE`J(A_T4&7X^*PFH_eBPLK0DF|HKB!b@Tp> zJYo)bJf&hg%W0LFG|p&?fcGnY(!ur}c?6^Q*#EgX4_$qwl<1 z+YXUJyC{78bUi+F5JHtZ6CwDSGgK_;BE0my!-%d_PXPY&I$l-jB2zlFc5qrJjH|y0e2ASzoiCyU(OiADvOdp1M7}HKZe%egu zUb$DFyI!0|TU}`&7n{QM&Ym z&fM8Yc8X@8sbMgga4&*a$vxk5!%KV?Xx{TozaW=IsiS!;w z!TOuSME8pjzWaWV24#N3-iTpprs~Wq%#eVTO{>7IZ-g8(pF_%rYO&-n=SBa0oyOND zkX5b}aQ70g zC{)IH=5KL6b{p!}7ewfh$d04*q}N&4ZUgw-A|EGQ;r1pUr9ppwI`wsq2GsCFp(nSg z88esk>nx^xtx*&kaKjS{2dGs1dh+dpDNHOmhj;7pvHHLmxt(>8NuOOJ7#WDf&v{&y zTH6~+cKPsbg=%oLQwu#u&-|L6S z163XPutkMPVm+JiK&sKz?H|Kv55ERuj%swb#kKR2uVT8|sk zJLuP~?yzss43rZxCFSe5?6v+X!94EwUvlOm?zrC%x5mbSaAz>>%#Vjde|K^_Wo3H3 z_6|5*lI3sY?)-jV-{TC4wREs^N}`x={TBk@wGHa=)FRHH2fs zzYxde`;5p+lS|a@>mJHU4C!$(b?^xE!kA`VyjOM?U03{}Ay?-^?eiu&>CZ8I;U+FH zx46l!tWBV=>ps(;ZzqvX?_Q$mVn!kdrV_(%O*HrSW&5zMZW-2{v84VN=8^d;Y}v~#8N5qEhF}`J89wXF!Kxu~+{|$V zXZHFri-R^$Zx2z>z0UQ}gtN#KyM^?q*e7PFO%^6bCDW~!Yr#eRDL2cqApTC1@Ref* zWmcN-ThBGnh9yogeq$0FI2a6ikL&1(H9f2icb6M_3v0WK0EOa z=FO%qdArf=jw=3;okW9#)xoj1#Hq3!>~$ZqUDtBSilcrU@6Ll~ z8&}O%)lY+uA0620mko?4K4t#=n*s4UMPydbBXVx67`l1V(Aj*3c4_Uyu)Z=%>$s5%-+Xf9Hww=dUGx zaTdTKi!jTljh;zehr?mzY@H%Q{D0~Zv4J!^@2ZO*?i5l7gJ^cD=v^9kfP239I|u?N zE}`Xt&*+u+KgpDh5!mc`gA8@g$Jk#rRPyTyy!zZ2_}LTi-@+Uys^?U1Hcq(w!z*fk ztDSb11k-;_YUr}h8;BoSJL0qZJ2WmY|O=}#n;_~Oq? zw6r&XDjzeKj(2v@l~@;m#XWNW}m0ie%W#y zuo_y%%`7)=n~b|NJjfnLS=`NPuy%__>8`W~)Iz$D8CvN}_j|7+c260UKK}^c3mVDz zM^oTO?s;0VON9K@TZ-r*&0j_&(BlxrZI{Hs+I1GW94*R@7Z|ala$cn4$zgbQXA;pe z`%H7+H_>Y|UekfS$KhX)4t~)PWgCWHKoz(2-@itdsPrwTeH!J&Idctp+A9GY=1k$| z#kb-vb?%-i{FMBgJjST+N}>CYmO_=P5xlB?MYdTy0mYUSsGhtX=cKgYv+_7rNN*7q zxqG8jdJ-(}Iz)d9l(AA+l#UzAgl3y@IMmufFIbBb=Six9`Ki|YNh%>^FljtE?a+jE zx&ZPGjiiyg_tkxxL*F+iQL$;&MDcbaaoM56F&BG@&|!D7%_kk|pX=a@lIQTmAsM(w zCKb$GNq_oJN6maq80GSPswpe5Q}CS{-sxkmxpl(J%ZxzXn#=3Be5a9rlfX=I8o1Az zhqnGsa9Ohq!#<0`_xc2?9U&?ZQ!m5E5g+N|lwVBI&UPmFjU4H@9}L|;B*~Sh_VlTH z8yWkr7dBfKVM4wmn9Vl8D^Mj}Sjf1y&k7-V}7S7TN z0mJWqcq^{uL7mV+V&maQJhk`YiUobFk&rOY-G*a2{h2_^8>L{mLJ7P(*hqKY;<|_< zaai$Bj=#<2CW^RA65|E?pm*yYl$3Bqy|cMEu%B{o4r=O?0PFpEb< zvq0nX4QiQe2C{$mquty{ny_pKS^7*Jtd2#3Y5NhJob(V}_7%afaTn)^l>{%_7`pe7 z7Oou^1bUT45O69Jc5ev5>Yk<8qP?5Ab~@mx_r*lQ{t|n7wFCwXPQ+vm9M6=84RJBud0G9;G9!t(?7$tN>+m?_hR8vGSx zqtGZTS*68!U#G*>bFbM4PF&XCU?$;nETtElbG7Y35G);>uI@oq7_^j)|nd4qm6;DeW*b@dc6Q&KKl-XMK%TBh5ac zMOo|1ux7M?Dh-~(MGh_0sYo1mwQ_gnu`0US!H2x5EMa^dl(A282O9jX=e*K0p?leR z+}Kn~kNF=bbAGvy{)viA&WB;t>GFpu(o;dWKNa?m&j;J5&)~!V6rE>4PVXDXD=nG| zX{iuJG78nXuScY1q(xCEqs$0d6{Vfhl9tg>sU$?>Irnu`6dGhx`H2XLN|Gf0=l`O& z&zqj-KKHq<@AvcBwhv}&DMP^o1cwHdJ{JvmqQoH=RAL{ zjk=s2B<{01*v*lVcpEeE*QXe!Zmu_7{puWP!kM@_26>;y6@K0{2A({WreY#T>Cu?y z%tGBq)Y_TP{Z$#j^v*UqrE4b)=xoDNxgG3hiyHQ)gD?8(*5j$2XJBo3G1QBy)8Z?= z6kc7y01;DC_w@jU%wKq5h8TqW7m2yvnYg*yfhpmgjwPQp$*Yi7?xS zS;tPHiAFr%B`>Cja=wtUr&i)VO#_lY)fV^pt_F+!uOP2t9NskI9Z<)eHB zNzHnWa})ld(3KYt+&Y8@duQP$g<7)Dt%r8cT#%SE9@Nlgv-wjkTcy@$o{gx z@CoC&lK3t%=#x(GOWnmI*CbJ2CX>Fu&3l@)`{*u<74&je8UFd! zVng3H&~t2KDr2oNye5%pe8ca&y;tCgdwfRMiIVZPXR%7Xf?jZx!vhGb9cpneHH{X(pvhU|% zxKsv&W!Gc3L%MPk+d^kh`e|KKHoN%&G_soSev5 zNp&*2^!|ZQ>13Q2Z;X`nAh!q4!mCj~!DrF~_OWy}R%b}E|Bc^-7aq^W8kGgCOYKrT z^!O4wN*6HUHI{JO`~VZUuLd>hUqi{Jxft|eCng48qBdJ*VB$zVJ+pTgoDQ0R@$oVL z?*r2JWon$-4qx!glfY5~4|3Fwhmh~tWr(5+KnP&n=^IF_>HQWNh25{sn?DMX-Ekk9Vp zeU`OKVyHJEgnQd;fHD*LdCAUmL<8O7>Gm#=7PF*A6V+)a?*Lw+Cd)loJO{Sg@_eDN z5RjXG5gjhY5(#q~++f51@6Przi@xlpOUrhEY3MAR@#QG}6{p3mlpdsuf8IcsQT+Tp zB^DI&(n0g|1DKDuJF40e7?#9N8^MC}pJRSQ{;CNnj__0A~nxn(ImEq5A1>$ZY@W(1e1*MReT zPGBvcCA@L}9g0f_K+Jt#5V?I2?8=n6)E}Yb_M0mxv3CL-o9%*M8^kHya?#WQd#H(} z3TLD_6Q$0_;F#ty!nO|!;a2=)?yE-_4T^LD`Myk;bE1&US^keHpK%G+_zU2O=~W2U zJd4^A-p3{U= zJxj^Ru9M*9;e|$`Wz_B0B|OZXAVqdIxWufQ^z2(lXT@v7vGXgSS>6>Qix@aK|2ArW z?j;!?dk?4glgtd&Y7nLBmFRQsSirtj}C6P2oll+Dth zxcE56Uoyro;RP^tY8g>|z7CGH+F|#~Vqw(t0opdllDv;bHnr_5)Mn3w6DG>oT~$gR zCU4_hUa;`PQ#493H@nn(6cx*_p@obo-u>8!bwkVG-_t%? z(e|52x9y@o{C%ju!&{VCPz^`LRj$1yASu(B$Kt}DtNS68^#zrz%JEW#8f|<99uYrMbgK%=}^bF9ulSyvR&m-bz@<^!CE{qJ&geyalLg%ehV0QFY=zN$+ zzV?2l(FLPufpt1OzovwD*Gmg}rv*aX{c<8Y{N2o4K?JV-;n^a}>dg3m?WnL)kJ&nV zBGv4UgBcGsh?M0sSgTltB{$CC@0%egV|;{Kd>D-r6wl-Os*`lWJY~2Yvk@INzJk;@ zA=#Jpi%r*8L z=wG%$CG&Y8*VTnrZ!}|HLllGEJik3K1k75p@awc5G;|H3v;Ds^V|3%e$962YGSgT% zQ#uM;+)t201+G-}SBOALaRS;TSm2vR0e8pq0d;Olz=(-GWdCEnmg2FOI^~wY(lB}c z&R~Nv_Y`1-^*odlJ&)7xgwpvv1+@RSJ;=^f;eLmW!e62P&?47UVBdX_j;#{Gd(3^9 zxil5hG-L#!IfeMBE(^4k!=YTQ7am19kbi?IknqwI^oqv{W_bT2)7Q6>Lo!{^cR*C& z?ZSJG4OHQCeim1#7mW|LsB)>6J#AfB)TC)TrQ+3VunDO~qtx1B~x)u2PpgWwpJPsce zn9vV*RtaJ!M`Mts6^*YJ<5tLMl7-w~cF7~&)qHO;oEHwz1ouYX8+4Gk95_y=9_KTy zQNQW_`O{Ho{+`bN^BxU?QXv1v9F&hYL@k|fj7%2`L5m(TYIz)O7K@~HCj()@^UuWH ztP7j#Bgy#O8n)1B7feYuAx9ocp>%dO6T8p_U@@Pe+b<3ko%01PbJoM2mU6l$X)4*$ zoWQB(tVOBd2JHBnO$?)f*k8_ItD%QZDT_lBkMS_P;1^!^bif63G~tib6gb`%NE^PZ z6G62DsCcB)&UAT{&`bbTR$gE_W;|VeoL4Dax_;lMay|aMN zL6o%8T<=b@rF<B$MppqbCUlBggP#_r&cd*)v z&%`;!L9WFr++Nm3FPQYx*Y$CjC_4#T7DX|B>3y{9>nN_o`ZG1pdJU^~oTpFuJKJ%G zekShVIXv4bVrtr0N=HhJg{oq_b#iksj#FNSIn^F$Y>HHJ>lJo;vjN;|-$5;gFSGf} z|I$D6oXLdOOQ_DJ5&BsngT}7EgRb?tdeP2NvsD~)(|JE0jVYTT}pLh_kOZyC#tR*pU-`{0T?P zpCHttg8ccDf!gOTL7nt;LB+8ee4Z;I@a-++qo+0O=~EwI*}!rfVb6fpUwfFmauTN^ zKOWz)v(V_4IPQM85G;=xg5#|ysu1jsBd*~PrE?TyZd#DTdKc)l)4sS$b}oL}F-+E2 zjmDo*z##_pJei_fJS&ggpLU z$P%A3r^#f;96a|{0n1$p2EO}`>fZH&qwXPeo-hJ!RvjkA&o7hBudB%9y+`@*&N^~X zQj@M+5sNv_c~s?%A;{azAxk2MSbxc5pp|4yFW7FS?L`BXp$+LM^`r!J{s^(NPn74> ztN?|mQu=zFrNHp40qLvvK*<1A?o9A?y!R@{%-qA4{NpnU)@AYnUAvVe+VeTp+%X+H ze-uO3yJhB*tqx>hMJkvW15HX<@Nz8x+V?tvo?X|3rpCa%jci9FOi(qSay}9I-E1928cgG z)=aF!>qcAfWuqKcU#kc6nrUEm^fT@oKNrk}KS;zMo=3Mdj6}8Z41<%sbj8jT81IvU z4p~xy$`ALTdLWbTkxwT2iPqfjz9_yOmkI1sebQ>I1_noWqQlP?I$mswAh*ewWoGID z6SN*4*2-YgTotT$dx^b#7H)phUMl<95dM0HkdLxvWU`1P>r~c3Gv8dKHcSM>J~(7{ z_g-IRdFnTIP`eq+SB+;wXN01)V=;A)Z($4;m(e{-gK*ca8v5QsnjwQ#JcTuIZXQcJ2 z4aaO;Oy8uwpwCV?L0G9k=rd6hyqwIr2_xBDX$_B1A(l+DTP@A0vL_1m3b5)D&;0Q| zg7y5IvAO9lkycj%QsNFzoT^F6H-3*W`#kls*pJl12R`WXJruQWc7<*TS_W*!%8Aps zRa4H8$=}x%dAADLSpJ@IDm5Ebo3D}JS-07UBY992oUi9)z#ii0Xs_!3UH0R*J!a1-D?Y2O|KFI{awFJ~+$dKh5ij$vHfOeh1=n&vX$uUe+Ck>L+vGog)1K z@~HJ`F38ygz-1d*LBy96)U(b4thHlNF?O zD%W1sO2xv;@mI=(N|Ayo^pSl6IWG2tz}6eEV|N|8HtFGMv9a93jk(0L(F`=IGwJW_ zDR6&(AD*A^jpw$W!`b&;&8qRY@I#v?ROgQ7SDkA?MAZx*Mx7xs|8A1>5k)R7zYY}B zUyv1+56SWs3|`u01J_Kpk+>iBSa!>eyyACA@9RH<-%BC*&Dl*v&tyYt&vsNDeHF`w zmeX3<^cd{PngiYs4 z&`)rF=P7iwj^;UQe9l6@2=S39mfzJT!-tk)R@W=K#3K?%w65dVbuVe@7k-x7GD;Am zGze$E=fI1lawxvIi>#5hhiN~fA!>dg$yJfTOK*OX(hgsGe~LQ#^<^VFZU~oMH{|S< zqQPxpCU~qjfqkDI;OdkWd`5LFee%E(1ANMXyVy&t)#PZWer5i3$IN1xDCL_31Et5zr_JyYF``JT} zE?`g)zgtfH0&&i6WcIR1&>GSL)klvRk!6zv0a_(k=Hdj=M?b-@yeWb`VOg|#+DZN$ zWDEQoyP7EbItbPrJb^ERlBsFZZQ?GNj3s|&a(AE{(xN`lu%SX?ecl77-(CY^RUTNL z&_FkZ@wt`Lx3Q-pny!?VL7j8M_)*~m*3G^AJ8Yx&;Qh|YERx^+37X&{99FYa`3?j%0DWk(iTPDAnM<6)Ek zMf%F-CRvi!M3@QrWOmdRoJM9qSKcU8w8*Br`ya8teK!iCVj0-K>%~o> z^^g;}1J8C1)5g(dv_P|t&*$vMtEG~#?+HU|=WKx9e|hjrE|(lni-8YY`gkr}F6q88 zC~Q4kNCNlV$3jEi6Q6L2zPpu7_RM%suUMWXRUVYy%a_6a_X!aA$_6%Bi$Mv`4L^A7 zBCeQso;GBS=NW&ualOjCcL{{PDbcM#8Z>$ca?6TYiS=c`I0DgPJ?xXZYpI|jBme`Za&$HK;0~XyfT%6q2>!%748l9>dR1OZVum}3L<;mO^Er{bYi5|iRYRvFn6SroQZkH zTF<|U+sDlVz1-2*F+L6*(^sOqk&rpAK{0&R1Lp7asbpBi1*gFj8asRs%YyIHk58wg z%DWnf8&pTMe}+|A`b1@yHYP5S5j@Q)fRg9H-Bo@@%EJxuzj^jpeB~Wga#G}y-6x~K zxP_ks>q2zud3LR79K+{X8#xmeG)w$jKa zVK8~~Mux08K=u!v!IeYNSZ1I?-(55SyL(kI=e-pu*)`C^dC%bwqs4P13_&Wwp0-I> z(&-1g7+bAWOxhmC_=`{AyS87jMO6W`t#Zi2XPeQcD1}ZuFo3p`p1?%C5&YtC9CvR# z#l5umBcUbTblJ`{^0ZkD1H05v?@R>-J@#UcOpByzJcP{tHw!S^;225iH~~G5+Gus0 zg&m*IqUMPqqOw7XD>hq!o`p5+z`YlE(QFnx7a3+E1YgmBcLwvX%uLW>4LnkV?7@0} z+}PAZlaF-K(6d@-VU~x{XRER1TLh^aH4BzhtCCnrZ@B7a4Z(+wauc$ToBok*#1~tA z;FM1&-%E^!x*v~(5&4rq>-H^dzS==6ElTKEyhA2$FQLv6Ms%FzQxdR#ALjb(rClb$ z_{Q=&JgzuPOP?4!d9lNQ}GROp&bHf4yzlGrzBej$p9{JIhL)1%P1=00@H`Gx66U(;Ug%WNY5 zPEw|7M!vZEkggSJ)Z%U{iSUl2O6l9_`2;`gc(RGuo*W~v_|FXv3|Qdv1xff*A(5y& z=Ge9KR`QN#U)pV}PgO!9@KswUwV1^b~k_M#FKfkq=7!a zciePf;yO5L7DNlPWuaY#pILub#Q1A}Y5$)2#2_0;&9O~DS3YJFzv|FqQqmy$%Z|G= z)rX8eJC1%GZIA5lYQ}M6HN+oiparWGahF3Q${b$IMelqJEPpq+-`~U}Oj!n}KL(mk z@9#s2j}E-AAQJOL2f=k?9nADDCFfI*q4?B8DAG=e(yMVy*VRbk-gt|22l5`sNDfCA zo}vD)&olpC{bEy2*^;zCp6y$c1aG}e@L2S8_$zG*&M(Eu7k)?o{iO%3Em?}*zD}qs zy_~Zvv8KTj%W&<3LN@p7JF+a_9WGuhhQ9m$JTrPe1iXuc!?Rtfi182^nrq9PlWHc# z0}O8b^?}aLj-x#f3@U~)VyHx*J^k5v6n6$nLuBQ1Qc-=6hNP&{BjM4IbagMq2U@s9 zwv1*^l*EwPvGnWCUHEC^EX@0Sl+?M+B(^;4HtMn>R9lCT`t^jqIDLa|ANg*!N_ZHGL zvkkD)O#?5C^w5!aTDY&>22HJRfmxmiF5-CvGXGTZrjL-%do5%(mWG1N)N2saU`d#O z3pj2A?;LqBi99`-K?V+})9?gKZswOew2&*p>ko`fk4!j#_G4}mxf>6NSg*Xm2j*c< zOgvNJ}AOnEF`u1+Me%6V{pupQ{-eDZrn z0$JoyN&R^TouP9Jd7CRqZl7?5Wo9~%Yx|k1a>2wjexmSq{uIGt45M2|oufB2@~G&F z>$rKhFICt%37a))=uWE~TI1MFXANc0irXx1ysyr^9cybAx7-(`O(}DbbdjaQm6(8mA71jjvAY{Su|?4fVEg0^h%Nhpc~Ub%$=d=Ghr-`J^ z8R%L%ntS@_1;)!nQ||&lI*rSKnqXxZb#x^(48-HA#%4_1Jiw@RGtl)h1#Ps3==NLd zN#V6C^b-FZEFHI<=6&fidvQhxDg9~GENK_LyRML$=jOp`^#GdhDI*+T#)5h3S}YZ@ z<=vAJa6_vH7tXj<*{M-Y92-K2QeHBA`W!3_d*=g9un>a6n%N}_%E_c)-Z?&ChTf08 znLWx`biADl@ec7td9hw{*5f!`l&wu?8Wqw$MH$#o+XZfC)F5PD7ENq1MEg;fs7O~W z)q6Ut@|CJG8tqR7&*YtWL@5T7ZKl8q(S>lB?Pr=_IpN>iEbWLcCUL1MP^M_cS&`8?6^#`l5 zCk@LUgt5{!aH#^HjpA95x}81j9BVW?)}I5KCt9d#n>@aIewtdH z=h+`7Kk3xhr|9VL3+NTpLx1sSdB76hWjk{*o($_{t^4wr)hoKG@%-&kNfOw@0Cu(rL3Vl%@5|O2o6PFiY2lZh=989v7f9Z!95PvO4Rx9W z=)Ce#c&jG@GUaZ;uAS>(zFRU*%m~D&P$h7B;)mCl-k_r6LI|03fJhu;@S3P1j`n=O zcyjv8h}3fQ`Im|tXB~xG4ME_xEd`q5_k+MEl7=cD#n#)Eq&aITZn@A!4OHa0CFecJ zgG1sRy8(pn-Xjt5vVbiMJ79D&rQ;)XNYM^`Vth@G&R#whbtihFp!5{I`}q`no_(EF z$6>m*R}Wt}?jehf1874<1X*|YEjj5NMiioMGVX7R+4+(pcqVoNJb5Mt`LpWTs;y(` zNwHeu^?W0`Tu-Fh?^k12+wo*b2l=h6M}N(hB_BTDqITs)?7~rs zcuwI9E?tprrXondPNy%zRWB1zuB?^Wyu}zR?Bc18WGk(npN_Ua1Nj`etFW=)9s`Fc zeyLc`dOe$fD?bjA4c}|=v8FY*J8&Tzu!8Sqs)mpsiD@|2$O)p%b@7d%0ZCK|$4vEC z;Sd1P{*_>mXJMZ2QDl;LD{xOYIFJsdh>f|1%(6< zncRWh6{}4TI&a3!%Uj^l7Ejpr!j;VMs|L3aWv;_Z8LBP~!laQrbd-|?-2vVms}o-N zO?!X@yqrOsP6m(}#!3D5+v>mHn)q(D4or z&%>SmD%hcO5qAn4NP0;l9(*Z-x$6>`ft#_^-sBuP>wgqBZodgvw12?tU5Ua9;aKM0 zIZ6Y}G>E5K5*bx<2-8l;^7GvV^m$M(IrXQL&6zTn(La+4t&4Nm6OxOV1IrCaz~)l6 zP|<~{+jdu|<O9+FHr@?$xxyJD2_9^^uNt1aL?$Cz)2C_@-A6yLtCEDol6c zVjSz4xy5F*^iDWVx#~jy4cI^gpYyVe@5j2DOGK~oEaw_m3DYPP{+@e~ zRZ}~G57xHO{SM47m8e0fiV_)7SMCHhu}wm8P*s9E&Y`M<|8Z!Z=Ip< zhPQ#hPoEf$bwEuKmOAom9OJY!=419boHSkm!hX#~BR*sKsZdB4>;Z6J@&^_M@Q!e= zbgHOu4DO2i(ES|Gm3wLjVAc(t+fD+WP9lL4qXjc=a_qjMpH$kepB&D5#jlM_0IT;= zVM8r73T>t9E=<8(YiTMxV-GvSD&fE0rDU^|0zUJ)Loz@9#@-lBL9((DnWiZY3QjLT zN%}k0Z{k1u@f)gm(T%t#WW&CGNxF}@MZVZ7k)R59x_=U-@3R-fA;%PwJCw!_1oaS~ z`Pbm(J_8gTpGJ@O_?mP-aY2iTyK!QH5PcP8vFec?aBqa9>TC_;v+W~syPMC+UybH{ za6M?X!JOS3YG?Ya)82Zb2@_~ zAG7IVsbV}5dzvVyuA-?92JkgX91D+x^0Sju{4SvoQ?3RuF*`Pp#bVOLooCV2stW1A z-HoiBUm9LwKhp{8=A*-$dTKeG265_DaMCmvGY=S1Vyg@44^3cXPXyG?%fawne8%*8 zBoS076Z>(`NUDD*P6!^L7u}=5DkKT zj9mP8y5>C3tTYUvkIhp^$^JAPwiJM&S|BR^R3kE~QuJ!0G;6&##Op=d9 z50PEi?KDDaw#>sv3!*A}`gcN2h!~cJD3IxM&QsAZUbrsVS$O62PUtzu!JA6~&@sai z>n&SIRnJcLknlQrlFn-a2L|b_cTF^byGGk>hN;!eY4oXBE^vOzT*2Ar>`JdGK$_Aj z8*XLeSRGX~YBD4BUX$Q&cPfvn%VLD{wS=hwD^XYU9NFcXjh$)wxJR>#b~?2(l`2{E zb=D|&{nDD;+Rt~vTofu-?$Tx^WRDW)HS1HCV^3k>j8yV-Zx}ORX^QpB^F_D!fy&@ZaPhOzETCl+DmlRW@nx!yoTAa z@(?WCdY>3w>mb~9y01%0cBD$iD)N4sAP z(UZLlHCunt{YzER@AU^JqGK0xc5g1wfiO@cVlaBz3dka=Xq@j!{vGV6Yxy(ca=sC) zElq@YVKJj4tS8gUtZ?^Z-bKY{HKq4B(A~?X;QHhB%tCiXT4a!dx))O*(XWy@;^&XA zJKO2$C;ZNH#1I{4#xNaxCtATe62vW)m{%bifK~;8;z$4$CR>rB06E<8F_x}~e}i)U zF?1=PL*A)|>o#!rik-A|Ivp z)8a}?YOSvVX*t6v({YFB9!$bv5W&q>*+ilz9eySWak=?SEVo^PzYk5oS95!WCIVl? zK5P8!CM~eAV_-`_EWGUzR{nPXN#nYeam~%qxbV<68e^4(wtFAI2VV`GrMDa^H&oNd zgTHCbqGD$6{g>oP!g?~;dlt97OM)q#THMX!#lQr2@}9~B>a+YW|9*NN8YV=N>Lo%V zy3z>kD=$(7zf{WAg<^-J7Zz(v!KNEmfh51=-Q><_wXI3`>~J~iYzU$%`=aox!z!|F#ymYj|J{#(c}p*1*OKq#Uh#kMtLX&Ov2qo4olr=wNFBs{qD?#( zzoh?qE@8aO7{SV_7$9$@Fy_#1@cTRk8m_8hyF(ymo(FD}_y9`V%W=QQO%bfMxMMcG zzLE9Sd58zTjVF(neIm26#RLQ4tZ>>GidWmy$=(HdkZXL4RGY+LLYV~Z z)?bSu0ZZUb=~x=RavomKTSNCX>SD~@_c*gHg$@L!5Z6biF?s$}m@Etd|GO86^R8Bi zm(NFH$MZC|#L_=TZ$L@u1We4x;JttiO!r%5fni-2YNw81t7IT^ZR-n~F}R)dZW*L0 z*%Kjtoj%6q?53OURpYMs=kPQ$8-HqAz^NJiBr9Yy>B_x>U#^Uy3o{+Ko?67Vw;i9*5QjErzmn__2}*uMvHZ^zbRXU%T8h~y)awD>?L}$cI`HfM0ALcz zoMGT8n%upP4E}V2J*$;qw%i{0=rR_9FDHTP)Ljs}S>8-=>>F~+8s8E-6@<`OC z!nU(He0Jy{Efa|+X(#pJ$T*%wvsqH$b4NsQRWY1=cL%W znQp2N7k=iQdJ-$U=)H52h$#_Nan2BQz3Ct?W)0IGoaB<1oqrErW(R6 z;O^*yiA_1|z8#;~uLV0`|MyVRvATk6G<;8XT@1rsK^xQh^cS8#SH&o^KGZw13TNpq zB2CgKA>#H+^26vmp+-)eTu?qfh`Nk9qGLIq$}%X~Swez`kxXhy$H`H}^g`uR>f@ge zsSggpmDB}hdyhHeozd;|&U6z{!TSkQX!#H~Y^LsFE|ZEi%42@B zA*?Z!Wo27K!7E_2p!N{Ym@bRJ;Vvi0cp6M+1^g9GI5vUJh`1wM{xKipQxkCL%O*?~ zjwV0b+~~R&n`rRMlho%zCbj)s!94u5lc>Dqoj1KbP!zw9UNgPOTJ!z=)swH{36Y7U zesMhVRU_ec!`q}QOdsS{t-`lU>cIEgVftl>A=~q26Vp-63qm{Je#^L$7$8pBhb7->e8BrFE2Gw>=w2J*mr(GU`3vuT$!6A$k zwwF~(*A?8=h$xbNh5Oxi!pq$=KGc2%ZBu~#68PJ9A$vOmy$n%dl* zPBqY<-G~eS^MHn+Aii&ZiS$1{OlJfN2xHewLQ>7S==+cGFVFZ`KCY5@|C)IIY z)D(I!bOY&`8cXE4ql~*$Ki*lb$X#lAPOpaEp!t(LA%Z_shF#O}#+yE7W|Se>@Jxcc z_DB~xR=5f#JrL01zN!2<1Gq^>4XU3uQ-f9In74kqV8@L(cpGq^eVqQ1d^6RCnWKDB zTSf~OKJSIk@D5UBO;E=kVbcg-!*?@+_ha3d+3p`H{3E!Vi=+iH6S(zeJ8_Pj19Ib+ z)4|nCFylWpL3Eux9X{}rb}c=H!YF0(+-N(59aF>~?L#E@%n4jP5SEd=4Lf}A` zELoZp$GdOK=!@F(}0`&e_;NtbOwk!bOb4%^UVkA`Z(v}t%OBJ~v4S3rosN;_|A<{@33K;v3n|N;4S4(r zj4#k;jXrAAH5KY$U$~lbZ_1gN)BnJ&ZW3I7QAcm8`eNYVDw^&1o9ZpehnjEiv9ry9 ze%{r}d`JYmS8pnqq&3XG_k2g1Rwpqb)wkIvnlrE|XbqM9*utiVexv?n;y7mD1Ua6t zn9NZQM1KVa_dE{A#Gg-zrEw;|Yx`RH)@DDM*-;EHBtl7*Wwhy}klEBOl;9uq7wM6MU?MpKAkl8*4M!}2b&pd_5=UekxBF=519_X6GPBaVj~C8)#dLb~8r zI!wCGS8XIjG1y`mM%)U90oyV7=%^$PhW(|#9dFQ_z!_BX;RdqlSP}Dn{Rs2)!$+cA zafi6f9)}(2=``(j1-%`bf*?|aMk;Z%PE!`ee@`LHRbDbRt7IYiQ?J2&OW70chTMOBgN*n~3GVKUY1|2=E%fOQA-P(6oei%!kH)tTpv;ME2%rCk zSlAzdowSt%B_GB+8zrI5n142Rzhr`Qmc#nPJY)Z{93)D6kjo3lk)6lH=%BL*M4f*F z+ghXX&?r5Di2ONdEwm#R$*M%UW-6*qkb#BwRfu%fUhe!f1x{(587H+bUFfEJ0gpNK z6I+$(IPU^Kk24>Eh5X8`IH1YwOVnI24Al-^RW;zg*q_98vQy#%`}(YP_CgNT|5 zP(D-xUS=4QA8WkGc>k%S!}baJ;B}lXZtGy&?#?0_Ji7)~d6Bi-#8EWuC5iq#8oI16 zz;@MP8m6>~gkDvLi8II2#WD1qtT^XzW*Pl-@&&7tCW1z;56KD- zCw_)kfW|f#aKnn-yw64+Rb#~QwaI8^YJ?g6e2nih1Sdm|$#$&PREMciTiHVA<9Oxf zf28X08)~AjD7YC{Lr#5fAYwnqk!7Zz(0OSkyXU_+Le{T z_fMDDe={z_E`?iIkzhfyYn|wjUjb>>QozKS)@0>)4qFFqLru$F2-)l77 zcH3=w)6tn~IbFl*I4{^Mvs(D5cZT4}-x5$%@Swd;Lb}iXE4<|QA1}AgB1!MV$^K3N zi)r^6v28kn{_=G+|4lz>sP$vT`7WAG_;u)tUJLV0K7;(lbI>xQo$l>lkCrhf(T}`D z-E#_fZeuB4;8`TPO+_f}#os|XLg4DHHMC|$FuCQPLBxrMV8^qKAb)BN{otHMygp1o ztDG`+2LHbG;qz%a);b0xCMn|m6KjB+JIKiAOr)FIPr>B^3HY=;ikh{bAw@;^$dBEA zBq1*ep17`ot?Dn$!YY@c-FgBmpKXW3|5mYI+wU@$7B0N z;rPo)Q1;)7^s*uS9_LN&B#$P?7o8!7f12phts(+{!AJaLb`j0Hcfjn7Iy#cp$X=b8 zOqIK`b7(`hEsa*J!vwz6q1!_?eO$-!a)b3*&y-qtU!u%#MU<+-rjp zcHT%O(=^5iOXTju0{BaR1kWT%7x^5t^96ijzl%|d$ZeL+juF@cW&)V5;~MqjY-BuJlgB%|`P$)%~hiKV}O2r*RQ{o9_HS zPfRtgCjc6H)0BtuP^}dU2mR#)u8zj=^pScx+koP>FgWH^;);aIii2fJuf z7}0!~!^UYWCij;wg7*tAk)7*{$O09BL@~ZdtRTVJFN&f2mmMJWM#1nf^d`ILUXHMb zpNYOzR3JZ(-6ub*6>-Va1auC0iYp{j(Ce%iIXm+-!%g_fR!aL~vXvOn)@t{6)Ud11Y=ZXnC%A6Fr&Z#hGOI~CasyiJ6+GFd+Wi%l#497i8J$FJ)?ih zEHOs;AJxx`K)ok=@If_{#+aX@0b4GC&B^UdrLP>2x%MFXZ8C0w>| z41V^sljo{opf~#-hRn3V%F-5c)qD(EsQn=_A-p%OB@=5pvOs*qnCqH&1AQD9vD;eK z0`@tPPcC(|dwvv#B=UD=XGIv8yB1e1^&pjdZRwanUAS>=7IRMH39Z$i0_!Kpk+g65 ztfBuR*sUW)5_14n9#tb-U3B?;xf#lJPZGS<5CQo^(@;+JH|5qHqESJgX@B*2;EZ3Q z{kpZ7r~Z?kj22MIL@ye=j-S6D5JQW9!A$MpG)Ny$Xj#@>vk3#p&rNR78Aq2B8MS8i z?-xbEH{OBN@c9M|dlb?qt&VhcT0c2$H4%S?HPZWfUck=&&8Bu+(Z)m5U_-}3#%7!? zY@W6XyRwgL zn&DcbPSR~#1a{0W*!Wr&OLr8})^ll0%D&I^v6VhN8|y^OXSRUt?kH+-*$cI7(!pQ# z3|ux0!Ut`qso=*V{QQ?^`egZF=3gslkX(ZAnnKb2KqK5t-$?>g9y9F{NAX?nVysfr zr3rQ3xRCF~X1(Wq2;oE2FrpX}U;iS`FQQHMKZ~W4n!IodmrLfdV~JV#c4F@lO;xyM zcyZ`7^cy#mdigsv@5N$Rq%#%|cU$1XhlA{~D^I9{p)0Mc34u`Sr$qeAC^%^)$4#C4 z1j(jJc>TRGZfM(26guaD+s#MVc+`_jS82ory{SkZdeM`g-b3N1kJQU=0Y37Y4CghH zV9UXWeHMnFE`)$!!zeCbnE8Y_;g72uEKvz2&0I=Y(yOjapshl~@Iv67vOhm2(86 zEWeSBH{u1qNIuMNc?idIQ;BV88ZN(I1b4X$sPsRs&O4sUH~#;Gkd+mR$|#yjD$aFX zH=#t4q^UtxDWNn!rR=>4l|7QmO2xU}*WJ(%5|N5_8rpkl{O<4fzwhtiZ|8CDbIyI; z@9}y)U(!dQwtOBRzIlLbW`>-b);*@+rOK`ANa2QtFXuEH-w=EG#hjnGm0np`Nan|$ zAU7|6AeC_noYJvq^3HG+oPN8UlSnV;|J$U)y~Lp`(_e+}l%K+lUpGQ9;b?QB$x7VJ zA|KAZNS>#CQ#h%N3a(r21s*e2ild*Wut)bhiTld;?BFn2en=4JL)LF!AtNCn;T5=L z$MoP)`+{{%HVpeeSF0ynt;qtZ|NYYq5iZ)TLqS5Kxn1Ji8IxePtoJOBrZsVT4KtW; zLNzlwf14~`9L-Nsj^y~M#e8MzH==1en@?UYMS_oekdV%8+|p6c`EKzpxVFpi+_w`m z_)9mHt*QrVxon+wZu6QVzTiJ?uDfy>sohb|g&hw=As9OaIs9_B-P``F65 z1HAInY;N;$#Ft!pz~pW!;vYgF^*@;??%fUxGD@k9nC&Qo_fnHMohh2!dwE4}_<;a^ zl=MEfrr3{{o;fJU#Y*|>Vrx!mss>l5QqF&ns3fjhkBQfqXzq!uJKMH)U?xaX5jNA0QjA7*yq(;ktd7V!qU9t-}`9}^s*Yb^3`%U5(mc(*NKfkc2rlH(5 zb3vTB;sg8i&Ky1){3cds_VY=yXSvDs1MHUn2EL^&n=`xKK|amQcWj9ywW)nn4!Z`*YOxcfVP|Kk-YGq&Nm#&0ZZ^9=rMz6Dop7J{w) zG57d*7QaSG#A)Xwad!?(V?C-pFtOkuKjaDH2KP1cyKO{7QNNNu_dSvKk+bJ-tFGYC zO|ZZrP{H zFrsswP{3hc;p$CJGIKu{d$a-FGL7Xvjf~=tJS$+rQj33W(o7DFp2=ljspOY#S<7uN zD&W_oW#OhR`D8_V3O5YgIj>*`uKYtdQGR60?bKbt@0w-CrBxTR64Mkuv@nJ+>r_tf zNiowfFd$MI?d(vCjMcMYv$)2F#cX5aG=65UEC0B8I)7_eDF0xjBkOM!lQf$k-p%6_ zo1}D>^vs@ym)uPxgTb@8F)xz1|E7)NdW8kxZDP)K9x>(fHV@(MCOfjr{#*FaIkuS1 z#hjB-EScp~gY>e5prhmt?h$^#zw9sJ?>|do$qOxbzq@LD=-=Vora8yBb%)b3}VUuH4vrk-U>hF28TpYCfyhfiJbw<`j$_xQ}^R zobtCRT*TizZc)5FfA#bv?zt=Cst#pxV*~;I=jcklG-wXbS~a=qmxuThzXG{~ws+BZ zbdR&u`9dr<+VJTS4_HcYHXB{OjVqhGh)kLJiKm@EAZ@16IVXL2()#f$7q~Hn7~hVi z{gqNAsQM`XcF93fZ7_kKxh|Mfd@e(#-5Y(*+{lG1^8EmT%^qayQf2<)EG_o>pa4kS zcorO1{oy21KXG!=R^0zAFZ6sy@-4CBSm(q#2=D9Q-yf(Z{avc(9=$%ymAk*>GHztE z6*(2;blY|4xvtHvQp+Owqdd9k-k14BiSpdhIpfd$HO%ImmKU&B4FZ7SRST&!SO8ZS zsS)!HA#Cf+KCaUG19$4yBF=pa4s0T$H!HH+*_C9mGLUQsb-rvr!Hs zs&B^Q(W_!ukjpu4V0;g^uHBx)8{bp5QN;GF4ke=2VVtsU1+S%hotr$PmnEhJv(dw< zc`wyqF2hiQKM_{U`>LMe*E**1(o;P7*ApVSMokUo+?&R23i)8MSNAwKZqr9@+?9>| zYKOmk!H=taUZEnnr;E9X+EYl~i_65OR1MdC)8jiLw(w`(tR&8^kJx3;W=_K;lSw{Z z&uN;a^Opp0-^RZGm<^ZC?@Y1b^eZxW-#aFpUEKx#fzl=X$|RnfJC6~~Z8ltWX)rfk z@*Ju$ap9!z=JM_0XsY4ZN+u-?(mfJ-xFOMiZ1%C@y(a1NN~NtN-fshY_N|zECoQ~y z+iw2WSbsjW#*}~S8p&pSE#ONd#&BxuQ~4#2G<&%@WK$^*Z`I_Z~(h54Mj~;(cCTN zBL7#Jck`mGD)BQfy{Lzqxe+lng<65d>!`p4aoG7lep2&(cGg4CwOO% zNFK_Pc+sAhoHU5J2mOEeK7UuuTsKcn;bRB(6D!qdoKkw3)6*>G z7Hi6YM^`pCBqEtvm1lCkMZf9a<*T`y^GkT;*vq_PO)A&^a}ZM15SL`sLu>k)`Q5Kg zNzLLSe%$yzQhTj{RgX2|A2t4E!^~@V?YT$z8-bFHoqobyxE9LSY8jzS`MbRS`WQ|w z(|}W%A3-W|GdQ`JFiz!98t42niToSf$|YQB;_Hn+lUB_N1p+g?3NUQr`XRw4tOa=)@3b2a|w&8@uqMH_Pa*H`ZD6?c9_#y#+oDCcD_ zhI8W2Qlvo`{&2I4IdRn--uGz;x21Osr^>Xr=YI(PD^_~^6R+~+9A8l#$Jai;&4!!YCp&kf@LR5z@Uc4* znSIn1J|oSP6OD{#)BN4Ina5^wF~&*Ug4PbUu;`8Wc5WGocA3jN=Y(*Mv7#N6aJ zIecVoAoCryo%7Vw=EX7+WZc;RPJHtn+4N)+_jh_GUnVTSD*p)!A*l{hevC_>*c95xNs52U6SUc9_Ml!?UwVE3#P-flh?(oM$ab3a%H^V zyHz~?yF&m}MhJbt|JUS<{+}jC_kWw5Adf5cylwpk@+^q)4zAM$n#mFJ`D73Msw>Hi z754B=yJm@oZ5`w^w~WA(9J;t}iz2@IK`9H{cpp#0_sF*^y}U|RFKLoAL|=G!PS!&f9Ekp( zhD73kw8VD5SnRR&Dn4?wNN}l4LjE?1qA%1)NaQkvkG->5mefOdbKjJ0bIT_R3K}SV zLOQbbN=F~n7UGKsYj9nF6Xkbylj3Es=+2fobld?g^jqpv^xReZ^YFse$ z0`k1q4~<{`pnQdmgzO8$-rG(#t~K!$&pMTh-~E?Kk9QcOea}qA$$wJmD9yWI5m5=U zX<9h@%QN&AyNX8CB;XA>$HgXIkMY&t>)=J#B6c-xG}x`6VwbLZv=6O>5B`JnMvN5X z)lNTnTfjnIp^I0Ha z=BP07kEzu7lLb4u&<_6GQHHOnheg55^PpqHGOU%52|oNuYO>%S)|t=^ex^m}uA7<= zAtt=`m$5W>K`EN7I1jwMmO)!=EZld~K?=?ZsOYLXb(=gI-YQ1X2$`YSY0fD0aO4y` z)h`6kl$ZpslZ>#QCMI-z-<*G)eU z7K;y$QW7TZcQ7@TU~1|Em=Bx@c9RYwcfAi}>e|haa%nCdArjPeC#taNFa@wXR?!f$ zMUgBm*@z@}dPDjB-{|{-$EeFJ3@cb~K$8M73}`vy!biiI`$KJ-q~%21Lv=;-%5rgE zhd%U`+JTJsE?k|hAdtW%abo`ibi;KFY!f{deY+;2`OejJ)hmB|$RZ1kORu3t_9=9i z@Z0>H)=0~QbgicWn{jt;e4|u!mU!y5u^=^4l{xBUvvOr?kln*mS?kGo%KOJSMKcl_ z7VU&epS`$nrW{1)N(oK}HF#$9Qfzc58Xi|&rFpXHxcVP~h|DxPcJC#)DceR%de-2C zJFk#uTsMx7HAeNT%u%r3cyvd+4c1QIN?MHtg>9xbIiOg8h8AgJ(_L9$`*IX4?9LOw ze+xyl_r9pTyop|^Yo|UFY*BT~Y-sY!#K}h$vE#_)DBkyf&Xh!~n5sw{{^Z~Vt7}A~ z_H2fq!*8LSJq@fcl;CxR2btMSQ}oYD68DVgpy|WA(F*rMytMT=Odrs(oVd4@%=Fj` z{WXW^llu#Bg}N+jU;Pdu&!s_8S{S={P#qn3c35nk$Qoy6hS7gGhm8-91vNp1EgSZR zc6}5`zn;}}EI(HCqUj7=+!T%9$F8IMuFuCu3l`z`Ky^rqi(^m;m6=;Z3Oem@B;IP zo7tgLdJtM04d#jp*j3I46?%$j=|g?=oxgxW{e$rrdplO8moKuqI1I$Szu{Z&K4%_-UB(D3hdsctL7rZ>u_qejIbz)&_|t?WxPCATw%-yI3n3%0{?9_PQl*3D zJW0dq*{V!;?r-`sLfG^tUIqQ-cCg88CiO`aA?X@hEWP0ks=FphBu$^ecnN0ldTJPz zIvj&K{(VK)FLVgch$gVzt;|ceeTF!LPNoy0jB44=M!g7g$U0L5^|zcLuwIU}bZvvZ z<7aRQ*S3R(xKmWuJpc*{b+lbN0lJDM+5FK?%*INdgrp6EzE2VQbN&_WdhtO>aQy;C z-o;>V9}5XNQFwOyNhn@72N~&4gI}Zk(JcKi7OAk9R*jTnmY%~&%bla-n|&UbcgMr5 zZ$40#(EwJ9H!};}XcQ^Xr(NEt<4@WE?JHyGxMfGg`+X0y+*e6(a#}sW*nKFmdIwv@ z2yG|}C+Rl_>5wUnMC`bqw4d>U-Ot3((Pzva4!)&I9a50>NpOjF%|deF7{@o=g4WqZ zP!YRJ?7Z^1xLl`SWPSb!dpl7JO1;j~oy!K{%|a8l{m1Xd=})WZ#}_husO?gwtx!u- z7ra52evBlO{>v0pT@`Gvoe%N1#CYA5c5o3W1cl38V8u)ucKS7A4&7_uo|?dw`>g_D z*_HI(#W|o^GnMAntP}UTkA_t{&jW5Pf>Kcv^*wKdmHI+)=ft_}7)@onxE-V@y9Cvp zZ^bXP=i!bnRdRgNbT)EMk$4}{6vgE+c&dDa?!PcuNXH!u;luzImd4{0qeK$3q8Uo) z5S*%8i{d)hp#{-P(T&X}xUA(MglClE?|ycWQG`iI;Z(9Iq?pe7cAFCB%C6qH-{^08 z99IPd;|oUP+3Z18Fn<354OweT$4yGW21lCU!9EkHF_?)z{M-t*8dvd^J-+nmSbeaR zG{MsA&eM!|ZwQ&$PuK2?z*5qZq$fPI8uTes0Ce6{jHI zX&fjx^^5oX^2Cn+obgo!UyH(9i>Q8kDZVf|6`S6FfS2EpB(JiSV8B6^-8J`wwgvI{ zBbY&wojP2RipAp(?IB54v3TejU0A(uGSr_EXl}S1>$;x@tE%&0$)@8R6oPt;#$mS@#24_5`T@I!E5hlySCG212}r6>K(1b|v4h>0M%lqSET31)^6$io zKW&l66Es@snM1x}$J#~|xE7&yMP+uj^1G7XS>Ib*PwiUEC6`3pzPegzImlc@V+C;TMC7G2)*Tr~Bv9KLBQFeEl0G$h^XW6FduRglI`=lj%sT;7mLG>0`%rYxIGj0$o6%Fbo3M_A zrFc)Pa6UhKPUk(Y!!_56pswKxPHh}RO%@N*<(6tN^+h;JDqe_JwakN2{ZnDdDS2!{ zoT2sgW0bIe1g!IsWa|xQfsNKt@q;!wv16hlb5R+^Y?{tOY1aX&lm8Y+8t%g1DB%0| zx^SWUZ#r=7K6&%P5SR4NfffD5P--@j9Xct;{NJR)sO)iM-|twl`?h3klah&K8V`#K zhOWg&8ZdonDV+7HV{o9P3E7-6AbhsDaAD11QJC~K98_d>Ka=iT{;*? zc5MXZ-3|0<|4cmf^%i(KX$hQ4mxDFk4JatENVGfZ7h1J4oXN*G0k<<8&Zo4|g^KPp z`086ce$*)#CpCtKqw}z1$`M$vwMujq4V zkn3w&ABkx?11(jh@wloWFcx8V*A1`QXSb=+*d0` zuX3Hm+-@OFJY_noxopPhxi9Ey@fK$2JQIF?m_bwqYSF@3nxNTyT0AiNDC8|0g`b`Y z!f&jO(xGVzEOVnjE;!dh3e%jiVz)cqJ}?P4&!|FHFhG(6~?Okk~n!ebF8d5t~9`?&~e$PsX<3(r*Y6xephJ#b7`{qX?!(C74`R1w6@k|c_ z`nQYwJRF(4b~x2+T#hb8e{QsI+b^~Xq^Mfx(T;jmpuu}F;)^ScVdbgwbZW<0n5ZR# z-n~wQ@;GDqEp8zlkfh?djhFQojsoNqwApqun zdjfI;qw)Q*D%gDJS2WF96F=N8q^l|31it+)7LCz|WMy*_=X|5lT#k{bUnj6(h#HRe zABi6{JHzIOQ7B7DhrjD!NhJz2slkwYXy)T4h;&awX)lM;nUV5f_lm=ji6vNLWioy! zd77^Gjbulk zJ2pb$JUte2Bm}`|)Lj}a18PRcIY`m4e9!L=9>0h9v zbQba+>4-az`a)OCWcq7;8M5Mf?aKHx_^tF2x^hKSXH3_PSKwG=Uq0saNNW((l0xa2q_C8WU{sz|KPbn3+LvWYZ$+m(up8!irgoNf-c%#qZ?G>AZoE6zCQ4P-d7(DsnLcccGf|b-?M@`_>aanO3U%z$%}#W z8ci}J>S5T!lVD-=7G~!q;~rN-ddPSX&uwueP5B;lxX!h0nMfUF zOX%1n6MS?;GAzF(JP_h$!e+l05x z7D0E8CThLM!{#NEaL>X{@zLxmk$UG?y1x20P4LK|{#)g_<+ifon4#-nyx0cEIvj_O zhjvr9SamWHokoRu&UmZd*~YJfPjGtuXrdg@B7Tm>!83!A&=!9LER)J%-q;As$Y1Ut z7m^6`HQtC7t~-O}z7FhU^#DIKJ4+W?hG8>fU7R^;DtrDdh(`TE5U~6U8s@i{cIG6) z**~MP_2>XNIqxr>On%cByY8dBPFw2WZjOInH^L7+9>U43v)wQ+@IxoGOU z52CB`OHrl!R=OxGf}HcOp@Vbk(Ve||INDd}Q4F_&*P~M8KcpN_U%CRcwrF7Mea9%3 z%g49+_tMXix%h1hqx+qU;B&Jitcvl4dh2D7k{oZ7~-0K&c?3n=L()H16!IczLDoG=z zJ2GVjO}y95gt$1lfc?<#D0aF&{yel;^x(v4OY4U%aOZ0vt&cW_)&jrAoWnVUI_rhd**;O4n_1#P&dHn_UR|RequOajBr&pajzADVV08J#c?5@QE8 zAe+}ZqVR-liz9Wmxc`0$bS?NqGjxQ-<%AwF+4Wl-v8Ee5JO}ZHtuk=TqYTUbizC-E zve?U(pE%Xc9=k8DK+~P_@Ono%*gMb!mDPd@C&C6V$OtAc_7>7*c1f`IQ8gIU&7${4 z=TmQ=Rj4}Y~l6>^~ev;_))BCuvVh!FvTkx%b zd-%etARMsiOk+>UADnMu#5`~Mz?Y+?*v;=Z+Vt!Xe6@1No07E2e*Gr#@zb%`^tK;9 z8|EVl^-2-P{0M@4XJrVSIvP^WEJJJ7E~9atIcS-3ANt^5f@2OU(OGrMAZ01RO1oyT zBYm{bh|LVuR~Nkzz2drrBYDl{uwnJjrwfn8R_ps!n-Xw!%s9CZH#ZoX$lEj5n7 z&dbU8d+8Y1*zJl3hFU|+;%B0ObMugRgAAQMaVP|DlWCOxdxjW2TZ64vHVekX#dN~6 zyW)>o?dXyJcQ|#`1qv)3sB8FDY?e5Te|BjrR?+_tYfXHEue=*Uq!way;ffx9u``VH z?Wlr}TbsmA?>(sAUo~obcq_R++z_u<-zt*dt_#n@M&a+)zWB$2li>0#TQq09H%LD? z1RrEaz!bmfEGA2X^-LUOmS@hQb;tARWe-EFp7;;^+g(VU_8h@+mLUXA?U8sO%xllaxi`MABz4gY+Z*SLR%CA@aOiat&rgN>)>TmCMY2?P1w_+9FH z$WL1c<0X$$lc#blFf1BAJ{8jDPfsB1rawI$8;!~XhO=JWMV*z_pr6NVScp{w8qj_P z0o*TGlX4PmiBn_UJ@L3lP6ZstRD<`)zbMB$3SU|D5d8l@^udn@EnmilSxjEohO2Gd(`r z5(fO!(LSv>*j{@arQ3X@D~?A(#PD!PHH^gh5l@l9%L=i!OBHNcxDWO3G)8ekN*uSd zjI7l3!ll3w6B}I;kZcScOSACd^Wz~(eKxLtT7()7mC}J#;cV8fjrjZG!G>U=SG(|R z3xs)?JmtCWK$zhb)Xo$BB zN7!>Hwp?)KAPm{RoN4x~f{hK$me0qSp_bUIENzk+e)GAWu`Eg_aMcd-kWmt%E;MkXi@&q&o!rRxM*D z0`+jhCKDWF-XhxOEe$mBq4@3&X?&kJ;$IJ{sL`@>I3&S{$PO>ZD-w6p`@wE_+>!a{ z!Tf9Z%&Y=DzTUc#l{C|Y@JyuKX9-a!kI>om(d0kPWgwGrgtoUgV@zxV``3oU^Yw=4_)F4I+9CP6Ci)Tc3e>nZ-eiD1%aK)8ZcjNf8oAHw~GU9i+ zktpHYDP-IlgUtH>qSJBdU{haZDgC+vPWjx2rYR3F9VXCe7g?jVBnqc}-6bB)2Vo_n zxAdxu2=DH>kK+9H;=U&XmcJz2sQk5VI^%E~9(bE04j1aDk{=&Ms&S7{xJL>Y#g6+LU3QeT3~izV@LKLc{s`H^`4jI%hk(iRqd-iysDOz=FPcl27X4tZvA9dCZ( zB#Qq25q+8xNA1(!(EdNx;1%md7hB$;%6k!cabPqUw%g$Ej@qnEMiGWyR6r%)H-p~k z9C34u4sG~BX+FK5w4_o?vvpg!pe znt*q#m4`ic3SfRO0B#jr$IG=GV0z?E;6Jvbbz_dg#h7H6G5Zd_eIfwjuTV0vMZx-VfNx8q?W2vvEfz`y1ghS)-qZB( zEkmsP_&BnwjzFSCZ?XDbfu49-gghgw8zrpTaHqsOI^skQ$;z$AD}|X{_3kX}RjQ0@ zkF3MQKuA}XmItjf9JXDkhF;EAgj@+Xbn$Nrk`6qHDs7_hx6e0m*h7DuXhx}%Ycb9D z+d$1H|3HpmS`Z-@0S))!8qW@kg2$f|vD54mlxPY6=OoZBg1^IiRtIW6#-!|l8r?ko zI8yW(f_7_MrAwQqQmMH?DB(&1%)5J?rQVAaUSy}JO@hIq1_+jH>UFM0Uu+zJ$VNFIFF z7O^h^UEOWKPCR_~T$;2x9)7*;rg{4e@r)1zwnK1Zef}f5^7LK&N@pF~`&1gINSTnR zvo2J1%Q9hR9|})e74cF*8SZ%fw0Oa+5BPSlEr=SNY1dpo$e-wp1A3G2@>f4Zmv_g2 z+RAz)RjR<2d&yvnR~_{H0yWzG^c0@?Efm)29H6e|Irxp#AqZLRkMXB6sQ=Bm=z*~R#A)CKzKuSg)UeTOx}Lc~~RAvCX(fD>B}ig%t@vm7?t0dA`H&>78= zxN}DjHXdWcy77K&$`zn%P2uqPLkbFS|0R;2ln*O*81PPK6!5_%?JT|Z1G(V-Pgsuc zho$rq){Gg3d%Pwx!(SI*(t>#$Q7eS9l||6jRtg)JYU6H|pD-fp2|NieVqTL~v0m>2 zxV6~^;`+5PNgTpY-`@h879OUfGjr(POLiht^Z8&MTZ+@ahvM}Y2#AGwJx)G=JJt-K zua4JcV)P0K7}W2&?+1h-%k?aL-zZlN&BK2FqKr zIqq$=Q4mbUbRVXHrg_ZU8OGNbTy9-R)O$kO!FM~0CSE<9kLy&po zlI85sK)CzPgx4zGiq@HD(s7w)_{ju2k^QNYEcC+&zN^KQn^*da_ACpbHWF*ts`dv^ z+fjt{iV2^!><@jJ+W*Ltmm^M8$7>`zI%~N{Y70OvtLnc?U-0>B22|HZ;8>4!tbDa69rsZd*{jIo)i!_N$c1nSd6|W~ zDiAfcIWO8+@w3q~XAu4U=|E*v+i-2MCxo2(3pbWIq3y(gte?9?bTnTFLZ>Z72kmds zJ-^n$iMN4x%(w9@>h^M0P$tRJ_BX@8867--!$BmP9|eZ5zoM~a+r{;-hQg)A$!N6P zKNu9^36^!G;MS0jkQ$i<=NHE#zwK^l#lA+^W*kWO3=1N3LJ(ckGScJx2(U63s<$llLW=q8p;N6LpZk_B?T4P=X_4cB8Q?EYYU?5%~T^p)NU)fHay7(PCJF8{TH1 zRcqE@xAyN?sxVcgtD_8ipp#^H?f?$4d_4t|(LS$huX?a=$e;HJc5~ul0a*Y$fj8zY$I?=%b5eo(i+fFl>{2k_OJ& z1ly;S3a%CZcW1*3zJb=porHQE`d+n(ZCh& zQ|Vii+sJX@Wk^@}hOC0sNq4|FW}B-HIX1gd$9-p7AF-D__dkgyIn0N^%4g!tRDakq zrAj1!+X_yH8-RGnD|*0Cm=R9g0uy{K(0B(C^i0v<4J)S6w6|e++og1@_wywL*C9H` zDw{rhSt33|g!3pP5x*Qi1rnG11AG27@hGejkDaz1$0wvgLf&iq=Y9hHbYCBo>-6c> z(o1aolMJ?ZZX=2^`-jB+N3ij=Cp5fE2H&Jvc;~OFc)Q#tIPS#Z`p;kBTe%#nS$+l8 znXR>`idUhdqsN2IUJcy&HHl5RGlyV@xu;Vkj;dl_-)SOuScVG1S#3~pkoYTW@DI#SR8GNw+ zk!as`MT{S2Ky&p2?C|%xXp2C4xskRD((j~0Nq{5y`&o%7-xSCp^Wu=wf00P=i-Mc$ z0Wxu6KAn@Rfotz5i3Rhac!q5p`kp(HuGhdqs*Dr;#UByxcHhQIVtznd=t6q2R~4%W z?&;fh3V81bKkB!7B&(YsW((&A!mCd`taDrpZhV%9<&`BsFJ?IN9#mk9e%1otpNm|Doy4icne5dnU8uSo1a>xa zaK@xEYSZY93p@^xQ67W%)!`s`E`JaF&v?RIZ*45I_Bfbi8^ZC2N8#vT5{zJ)O#MYG zEc?0)$F1H2b`H@v!PJ0B{+%k&sb<1k*9lbjy8<)bd$4hxwFJwzAq6FS;BWrBm1rlmUGg-@~l6VeD|A4*TWw6J~~82R5Nv zm}5VHU=t6RJ#93ccg=wK``*mQTLh`yrIyVlHON)U5%0{fWHMQ;>}0+JGdTD{9BpRE zG)s*rTX_vXF`2{kPAXxS2~$K)fr19Z-WhgFdJFr;5$qv90Sb!`;fa4YkihXNc%Ri_ zeC!qy)vBBlZ@je@>o#f8zOYjG@iLo)7Blo;b~751(F|jngYov_r$Xk-TEWKk9PRjnska5M!nDeHa zmQ{_b(2tw#g4gpaR-fN1B=FNm~`~PTWe~oDWge~}^e+FJBq<|Qpx#0b1I}s3f zLd1|5#e5nqL8G7@ks6Ao~8(3oYMtR6N7;93FQ0PeZb@Au@kg38T)IpoxW2AU)I< zg*vYRy3B6~&a<}1t8$*>RryHJo@JqK>0z)sa{`{R zUmNBPe8R?chjIG)a58bI1a(dLO4~=BqVbFTh4*p>zjikv15^PFQXL>DI|&}k=pZHG zo;Ymyh$Lkt6XW@V4CJT%RjHTvK;xVB+OOx zvm?=)`srxF*)Nvk=Np5yss^@mJtJPT=m+Zfbxdsf?Jrf>)`{jl*ai_CVz-VTLAKHV zAUOIa?QxBuh7CsGx^lVL`RhKKRDBX^hFGA_BNLJDI0-P1DWRGFslyAy3vi-yJ9=lC zD)K9BhUT6-Vo&V>k$$lrwe<~b+$Y4~6+OL6SKRo3;-(nk)@lin&qf&-V%}?MsH={b zKHP=+cwM3{KM`L4*GQ$Wv{(eH+YwS-C^peOghF(Fh#m6m;bD3iJP7_RuF$NX2ST5ThSK6#emzw^QKhz?r#+y;yW&xrI2J+bDTQF!6eO;~gP3L)sL zmD*jG6notm1y4R5L)Lwzf_||AG75@Bz6}%bop-UY^p-E)-)crH11{2T#rf!K&=~l6 zcfRPShCX_=*O&%$uf~q{nUuFyg7Z4dNOyb;Oc^yA-Q3epcV0U{^`?$Qu}P1xruRBl zIAI)G^PGwHl@}m|#tix|k69kM`y5RFREvghUIK=D4nok|)pTXt7@Rb%(c)+72pBH+ z07a?%q-U?Ti_GhH!~4HwROQlCI3I(FOvqSB%}heBr=`%Mwnm&*l?k3N!$9S36l7nu zL>V0p=oQg_jZB>3PfYK|(d|dtZZWb`K+?Vi2?G5Ogx@lSSL-o3ncd8)1}? z^0anhI9f0#k|mE+AYs>jgL8&eu&wEj7KnV&&koNc2G zKkVo|iF4G(;3k+%9f`5x8)WFB%x+!$0TyddVFPak9O~xmdmbrP_=w6_7*35n4I{?S6;d-I@aUX8dSlym__s+4$=uC?O96t;aE1!L zF?%e;ex3)z&)`Bl`kn=hK5;N6cNo_v8 zDUg^~|NDvFK4@gYfB(?P950x5DFMU=J>nrA1vIAqJ8b&n4M~=h;b}=MUWN}NJDL5o zRc@|@%nu4pt8YV~hY=h+nFv1x%c@wZ|7oqhMxhVEf2a4AbQir5G zaaX4nKilF3+UfpJz)Dg$zAxGv?|_Ibck-Bwx9HA zbz;fk#c*ZB^9J9KyW#DMekgo#1BM09peM;_JpY6oSH8HIn*C=^itk6lJ)Ib^>=fab zlM`Y5T~G0vKY$fn8c1nx3~n)CRMPDLY7(iC7f)}B|E(~fId7}j3Kc=8zSU9Wu%ich zMuijlQ(fdaUxOuCU50V)dF-K~9v|^?C``r8IN39s#Gide6)oz}knbn(lgi<^A$ttd z)$zue33B-P92q=)mhfHMoe}iNNo2&mzclD_0)Bkg1Pk9Hu}iL?SFOunx~Vxa$e6+G z`-{-L{=G=YHU{7M=t^#fdXvk4^Reih5)P7?MIMfh!r!YbV3GAtY(GQjRZ{ke4I574 z;N3ID5BVf)At6H&er3@4=||`nl}B_;uMN3el#TwJ9Nd;b3(6hAIfqP$cwSV zV5v~w9tao1{#Z-2w+*q!;vO_yVJY4*-VHA}H6FD+c#KpEd~CRwGx6`gEtU8N&=LKFpLyzg>=jM64ty8f}C7f|nu zBsB7UG2YSQMb_URhUYJg6+eElQLL!=2T~TSgOC5};7hbKNuHiF@e$Fba{MznrU5SA`H40}~oSpTM((61v)dKku6OQcxPVp$@{ zZ5nk}+k&pDA{#m?kBG|C_2tV$MUnPp*wwxj!wP+k$*gce5ELzui-tQ%9tr(BW3tLcD zOob@hbC9GZ$f3*&@#0UK!?E?RC_L(+0~)c_pS%|mvUtaPX#9r@c-;X9x-6{`3_TA) z(~_s`$*(dYU1l?Sp#GfxyeI?yYRMqvYNA;Qt`L7Z9QALbc;)+x;sYax<4hGPSW-O) z9Lv|>Aj1VDOWqWQj@<#z=X`}ozXZ5U7qdl+rnBS8Ul4n~8Z}k^fy>j=kZfZF>N|Qs z#O_$I^$!vuy1E;~?*d$zpM(#+Ye4&^T9L9dH4SqewBd`^d`RDXgk@Oj;aB$(**n); z;)H`7KA_*%I8V_82d~h;LETS5V%lTW`uPg+J-7`j{3+ElivlyD#{RKrGu%&YhX>f+7IUgj!Sg4!Qwk%~bWqaYjYyNOZ_61FSIg2E34G}-+tfLYOJ6E8xPZ_s6)Gc4-$Jwkp^nOl;MFS4td2b#Y<}eGS-r11!?$79reKw1@F2kjE=8I##Mp3zp z8VIS@h16|Za8k^5xU|;=EPWtpMt4++aw=aMqBOffLro!x5XM zh=o^)E%s}(pW~v@=ZQk9Y2H#c?sXmVS^HcZu~LFOMe4-kTpx^iZp1y3yh{5m;~;1F zFfcDQ1GBMzgwJp^_zpQjhnXp`7~k1&XRR9itj74}ynT&3m*|VE&L!YFn`|nRlZ`U6 zwb{ZeXK22H6e`}PM22`bfXac5@T3UiQLiG&&%2)JTI(3*|G*1hoY#z=fIGd?KLe5k z^5&6my;ydV5xThjF@73<7;F6f&8qr^`QX4ck;}AYm?zz6XZm93F8(c6a#)V<&f7z_ zM~Tso5PkAK`Vt+zeysTRdL`UBVjSN8`!pR{Hh@fSm*Mu`wAUMwI>Vt z^@nG{Vnazh>!g6Te)=;Tqj<7&p)Xmx*qKZ}(j%%x^M0M&z9Ar@h@TpdebM2t)0V!d@rJNUq51=n0DYLnN9H6&=XXJ z+4;tS6>z^K9-MR1n3HMJDC55wsyB$CSLQ8ZEnxw-SJ#p*M?Z2w@E-3G9bugNKhs`u zAJ)KRGAdR3(g|@hMOU6$BmF%c6+TL_!&Aqz7i#PwPurAyvU>$lHb;ohj?~(=2Xh5N zt|VUcE21A#MWXWHC{91@4ih|dft(I{#@!HzZS}9m!u*$CsLz~TT;AUkoL$8%gp^z? zysJw`o_k7fzv<-4ek<{B*!lGNlsm-kXcm+P-Q|`({6|_9j7g@hPi^3?a^^+zPa5;} z2(Em2zh-dP574^MPb6%5$-R|7$o(Cc$Vyj^=8O-3u!6mK$03D&IlRkUBdC{_9%&-) z{wRQyr~(q#%3@;kQX(PH$tqR%q8qyjBu7l9q_dte>KQ|9!vg5ccW-dhx0y82LKd#A z$Rua>PeEJND6-4Bj70p55Di~)R0u$tgYzX3an6w=3v18P4XqJ)BKQ;K#r4VGS^uyr z=mC+x7Kq7l^T_msM7BUq1^kyc!9Kl0+BDA?jJM>XY*G?bUb>FUhmC^Xl71+RNQKXV zZq!MA8gEh;PHev$;;=pf-e2-?lTRggugV%q*I(v}lx+p-&o|=y(vIBQ(oG+{`cA#L z`8esc4aA>v!`8tXA1Nx?tsS zR$HwX!A~7&lnbrc=u39T7~%WbyFgr1i|7>F@(GpG;JfcS`uArYDJnY;DhU_Cd)gkD zy5Khq7!A}$o-l>N0YPU!8o*fW^kzSJh63-?!qsS6krB9oOqyZ<2Vbh;K&~RryLO!B z!sM-bPn%+xIqobf4psKC@CBKNnb6At>vfB0r&M|*x<(P^yt)1I<#v%?mQBT zDf<1mSq#|hJKZX zkNO>OIQ|?nWmi78d)94;itOaPw)^7p{!wsxdL*^jQ$ZX5P3LYDj37o6UqHYM1sL0< z0q<}9r7;new0e{-EE@zP*kZ_hlpZ{9nihDS%y3!0II7xD(;Ru;@`6gZ6Qd6t_ z{We{A_apV{qtwp)Bx$~90r}qT)FRyk^OUEf*U5MIgiU1bcC|6rW{6>T^Jti)9zgbQ zRK@RaLy7lmRopjT7Bx!JXjHrpG4@d6qhG(p8{&rWR(l$LvuGzj#3#baYp$GfFeBtt z%6PFa9mKv$n>Py4gLA9DFcW3BpoR$*c06r%|-rdkpWpCK|G@PeMiY6M*O1*_&D~sKSp-k(Z4+%2emV()Kl&Gbft- zX-|PeX_2Jqg$~v@C_sR9IBf5x@bs4kH5@+*O&Gy*E4`3k<`7K|Zt`cl9~waTjKgeX zA}74}ZbUkEDg9+wg<@`gkn$vlY&SfCf$Fo#@X*nt763J`5p~w0`2%V}{(E&ck51c`Tnbq(k~Xsi0JM0xg$$OBb$sjALvhMFG=B zVm!*zYo3oO*{Xp4VdL28`xnDlgGksdT0#8Qo+U|PUm>*A78*pYp!YO{tkTY9gDV70 z?(IxccJc@3qbDlk zm=7Czsgs)=47<3Jz3`=qIFHK|=x-AEwAB*+UHXrk|1txz2A_xyyG6qH3oPt#PT+>^ zlOk)J=P;*&bI8zJ&~vClwu|Pt%O#GjM+IB-FaS6kX&MdB5Fvu|0M(b_fyH*#qjAdddyS8ZNFjEbSIK zZ1jW}i<@vG9XEomf)@HK6?2Q!cHn4JZ8*GefJVGZhrWhIWJOg5$qtSt7GKSo)sL2u z;#Mnu)c%P$+G#KE_C;t5J9&uk>OPEe$|U%kag<)nJVdO2=a97QIrPt(W>Q)zMRwZ` z(2~6dbj71>Ohn%qQjwNL=bLHZhO@;?_XkOQYrmOvD~!WPn>a3L!ddp8el*_M>Bnt3 zC`*Qm!tubdGE54Z!0zAdfO1D4GJE`!xN7ye@TEYKyJuE}_95S3+JRVz2$q2}4ieZi zyo@fnm=A*u3ALTut!T}xAxbVvl7>{F<@MP%YCP!cBk0lSRD>*~R^}Q~!+fC8=8~}O#3qnA{*j#NeN4TTchI#;;$1q_>J<6L;=Xdp= z=bb&vF*q}Uyp`(VxSb>T_rZ=(pDYFoPj#U}NgDHQ`*i%das!<2cfk1F#q5#v61wf@ zcJlI-I(|H(2nKB%$foo}5~c8!{?L2Ks|?$M4^KGLkypgvac&G~w^a!q>o=rj`Z%aK zH3bSgRpG+VR^pnjj!%z_0OLbZjHH@8>1)%4=(`c%UlRhUk>L=NI0H{syabs|9XNi% za{i=c4f*3|3!3B{b!x03OaIytE1AEjQMMK>g&ZKn=aBiu0{Jw=2R9xR8h_?xu)OPV(ykhU?}c4w;rl^c zwQB`_bT8y0{l_x0VU_stW)V*QucG#kauQi)dJ8;_i>SQjG+sUAFI^eXi#|)Tu^^B| zhgEyPdYdEY1cvcP?d|zst2E4Q&a5@I$%k5g4Zm31QD~}Jz>gIg59SO?@F@X~_<+sl z7B`6a$LEdsBl``AS8F&1scwf8>s;vis7#C!X^}6jo^br(YW(cdf_j6Ucwc=usIHcy zkMrh1#2a^h!M7%~6exfyLbl~!k%&)?yhC=TcJfBbo3JHI9KVcMk1Gm?^Je!BaW1cg zR?omTu49`8{_8P-Ntq8I{^~Y(Di=#;lxxsK7Fw*?+EYYlz)LW_u>Z(B8@&%w_5dmQgkfp^sFm-t(eVK1X9HMM-S;TEp^-hv%O16QMu*d3E zvBBW+>D;+Fog_l>?{Hy%>gxpWuhx;n?t4l9*?PL6 z@*)JC&f~^j`%O!1%COK_mOU3P%oc%Z+>FWnBGRb^VgjYU7~N5#qJ`<)`jt-2t|9YJ zI>MTWQdphY0g^V(=qT(ge7Btko$xq_Sz%7)h4ulhf%VKQ3nlhHJ$;`0Y6=Ukxu^0zoJcblYOmZz+rVIuOp(N`+yWSOri?GL77j5a7-(IL@XM$HO z)xq&`6`dnjLlYPAG;7~lXqlJ}pDetw&#;G?EW9t<>#E?3>PoKP*i^VPO#pj?i@f}c z`QR4q%a8kYgBObtzIPc})aZmcJIh)dEWd@}TB|ye>-~l`S_ZV|?F#z5K?=``_G9Pz zc~~)dB?c%@kA0x8)<6bkd`{`udVt0v|fAJoPvMT4?j{Frx zs9MANC-ZX=E3!^bb@;*#;q@gD@U~Xi!*EKu3S&3 zZCOSW|K6qF4tLQ7buYR6^Pe$xk~003oQ|8a2s(%v@ImW)m@)n`Fl60Cq{jYZ&p*&& zFPw>?AD2Go+>9dlfRC{-#YGj%whMIBk1~wUj5EwS*wE{D_IGf#;Ct~IcAaK49m0mpDyr5k!C!beMEAT<$I*L5 z+<}ToIIO^#^!X-1mbD|cMC7pKn z6LCB}7sNBBpq5&I;Ey;673)-Jio|GK>??ROB(CysK@H~h9K<(f$<(N|Ml|dE2zu$> zwA$^f%(=|sO+wS3e!UK;aX}wd)oaPF zU|HDPvy14a-=q(ll3~XtJMhf!0C`mxm?QYS_Gn7dBeJjH_CA88LZhX_+9fzeaV6Ke z<2=r^n#Gjyui$d?2T~=o8*doR#%t%?;mjT#kk}Fkhr$z4{aq9JvA-IJ-MP=mFOS87 z(WdA+Uj`NjTBFSD8DwJODKbvlhuWHkg7ZCX*!yje8oP!-%S9hfcbyu_d~Bv>$5lXk z%0sSI@GyC36d>EVoEbhsL$vL}dRQ9N&dGdohwS$(J$^_y6Pm5~@eemcL9a2hYNScc zh`WLYy4A$AOmjS4XE*g60Dz5M?-pJMaysXg3SRP+NT6q>T!WeE!c=Ex!y3P zZwpPxearZ@`!M@U*5LkGnK)xZd<9xL<}@6i{JLiOIWgM!qlZaShy$iMmrT@(fg5&pq`_JTt2byf z%iAmPj@~+~OsPcmX>#aeX~1^x5bg_K+{pczmVDM-5e;(vM7wY0z}%QS)WuJUZK(^z z^?r-_-KCkN?FWw)2b)ODOmU%A;|87m_asr$iet9052;I70ee^>kZ43t#+ZzqF!WQG zE3`iipC5ms6(et8?#kB~e(w=2>N z@%ZO;>};#kL_aZ|yw;}zla0p^Ea>Jm5m(8V zc*>xacH}-rv3sZSWVSKe<#3p&6|8}UXKsSxcWYMVmKL^|q`-i)FhlJ#!_8XJG_g?) z!tUgt#oAzW?&@3*g-HPmdlSHuTkmq|gt5Pk$ zsW=eSj!J#1_-|1!>+K2X7p4tcm|#d$7n;$}oxv>GJ}9x1=P%?PhXSwg+AE@+bc$Ut zJ+jM@ZyM8vd&it0k1re01v}bc(J5e{}{aY>cTj+gAcoTAAd-6ERlny&=Jl zn+Tciuqk;OzhmQR>N7rvSC&cUp3RLE$j2>gbn8muFL9l4{OovLpEI1yH8{; zNbt`0WZ|-o10A;R3aQv9Xzx2OfY^c-%>Gixo-voBL|h7f9WZ3AIBB$EgqcS@gAQFj z2;)yJ$B0Rjz;$3U?s<*OE`u|qTK^E8F~<@m4lKY)&kIoYj~&Vs8>8Lb4x%%8CYy52 zpHFeW$Zyh5CY(|@%-vJJ=`c+N;lA;Uto|xXwJLJRxSMIv;JAZ3 zI^rg*nKB=aA9ZKm59;%tYsxWn`2-8sbJ`$YJR8@d4mDK0h^DiK@%!g|$2DWO!MCqV zKs9<5pJU)i^*7jIO~r1g*11gH_h-~*j6Y3h6isAr&JCkUDPzDTX9Uc86bH?-*7Hi% z4fw}3lwO=34d-?@@&OkU(ffT8^P^Xh6pm`fHPx&5Rg1TSjAJEr@VO;Ql(XU#PHDlp zwzG7ur5HH;IE@nDbMc~1vp`t-hsjsMur;QHZB?8EMVVjO&5`!(Qo}Tqk{wG9Zrj7B z86@EKxI?5{{0-f3Y%7g7(qS{-BtWSRa*_%<`b0m#e`3ZLiy{8T%-!N%?&u9!Lpvs^$e>|ZJT&IeIikuEC^?irAH#150 z?H021*?;JfARV6Dl>nmzv7H2>nlJxr}Dj(w=i_4z0vPMv?z~ISFQcR$!A3<#Fe4zhzeQ z*(hR0!7Cw?`^dM8?98j=hQphWB51Tf_BWTfZyp0xH}8dA^qG#MnAL*=I+sig)ZkQDU`#}WHt6p+QI1F zW5GcBBRmw~`9FRluCYHMqLrplvdN2`H$n?G?TI7&3rptDvJf~F6d~*yRzaDpumfxS zOCR28!$_?te3kx;{Myq4U+*RI-~MXgGV6RWeppXm2J`SD>Jf9UI|Rb~mQqIjBYoig zOW?nZMDOovm@_t!1k5tSoj0V}dm%qSZq;)b{htSUWe~%D5q8sy@HG4T))5#!>nmB> zAkBw-R%IWz-=j+&uw?1wU})8PO~-}ifREx|=qO64U4@ZoQa%zs2TjGVXR7F|nY+PW zBLY|I?!@(`MO4No78@HEvT-L~v%i1Hvf07kMej0WAa9u+zj9j@P3V_o!~AB$)qnB= zJMbe~&8UQ>sa9BhWChrT$Fb|vWZ2|IZ(zINAK$0;A3Pe;X59jG=-fx!$hp?hVDo@t zQoR!Sutbg*@_(>lO%8Zwjv#Wu@8I!s9xhKjLC)LW!fmSR)Nf-gw^KNE-%ReMf8Q*| zvxcc8JG8AM<>rWBxe$iK~0^| zDt{*np0sZSvLqX78_xQ4o}fVuPPBBo34S#VhiPdNe9Om4 zkabm6fSp?4yu~>f)2)alFP3Bag0+0aA`1*NJ%YO|{b@r|954B}h(;dkuia@|M^5$6 zWpiE*ft=@PnmkpKk2vJVp9o=~^S`Ui&4usK|4%q-ZeK(W*E-Q(eSKupJC4&!@y6Wb za&rEgIKTD3{dl=?3dVasq+x~~U_R1>SKGdmc7M>o3D*?(_*!#p8~dFsdaD9Y?;pZp zf6}2rdJUWP`6yqLz6I;1jDgUYak%wF9`v8>r@IGT==+gQiunGV4LE$=A+~==8CN`t z;L8H;W16@+49)mJthG;56N_-1A6pLDH{;1SmsxC&eK^dWSHizZJ_=%f!7%yn1EN;F z2Tw1a2wQu2Ot7javosX>V!bm|B(zpN4;T#+f+kzzFa%{D8=)Ym8WINfgW0`$4Bhq* z+Oi*zlH5YPV8W20H>+TkWh_KZO~r$c^Ra8*Nx0_`gECeHjAw}wE~&4;DUbJ3H={t( zu)vR=gZ*HcwG@X1pQwHQpa33?UJ3i9-_Xei)}vC?HsCBn;NAClI1#ahu+3s1I`4}7 zD+@T&riC}Q_j0ygDm1F>DWTb&RLQ>>)g_*z*_1x~ZJI>t@0TJRPC@bhrLc4O9>Kr% z6vt|tVSiKyZaRMvZ*Hr{hBm=hrRWL@QE_B@S~W9%`hIR{;&?pyA(~kjbcxek+(dsk z?E{@ZduUju8qr}@u`m5PE)FaQsh=vWrr8VTgP@NZ-+dskn8cRD>k~D3 zXw1I)5)aK+&G_c~k$80BERbm12HRtIlAAw{kf$>>P;62?{5u#(^mav&p`)Mh!kSJp zx62+z&3j7aGY!GG@-&lLJ&5+qSWpSs51Vg(Vir2sLrZQdzccbMegCTz0`@16xIN?8 zl_ni@d)y!~lKO-Xe<^~a!BJ41bQ+gfw$Vz9EATVv9X-;XO<2Wb_mrUp7S!oTi+to$sAQ?q!>?a7zU08P`a#_aSzySSIQgYy;#9nPEoREld;_NK5Fm!!!o!?*a z*}PnCl&H{kDYK4BV4;lkWPXb~g>a_J+WcAF>wR z6*pO_`(1RZQ#FiOuE<|~HkCIS42R{SEU1W^OjDeP)}GrF~2h4}8!#u0Z^VW3$L^Fs=l>C=AVh8`u< zez6Q@Ih=+B;aSx$kzjJ>gko5gz-P5{A;wGdsg932TeIDq^EUfM%r664v`f+#Cvq7p z|14r4^_KqEL%`}*I=Og9o!{~*nvpbM7&Hq7@A4U_aAz+}(5xhfwYNg@qWSdQrwC4N zk|`N~U7nQPyaapgE12s)2>I~qELD7ei8^qTur1IV17Z4U*R*u)t8D*CsciS1LvYjTFzG)z2C_C!Mu}b#E1j;(-d>eYWyLc{+?A6+J0mc$uZnJ$ z4PoXiRpOV;5$2vd2H5X8g50|yL94xt!CT&s%&lu=$_>V$Q#%W?$McEnt;uw4*6iAx zBkq{(9zvXo)mX1lA!Ny>68bJ@HvPO=pWe7z4W~XdV&^wU=ED4g%-fDXbp1O+^2F#r zkh+>iqL?MPWyWQyU=j?`8&|h%JCQaf?aMzZx187l(bvrC`+Ek$lwM5cJA_ zON#j`IQ>xrZYr4x{$;v+s^{2C-ydlJ_dyC};-!K3Uq!#HR&MyCMReouL~4B66>FD% z!0TPkaH=K=8*Gv*$68756SwPhxGKdkDS4= zqg2Yhl=@z>gBdTkldDt4(l49eu#@y8Vdk4-7}4zlo0@a+?axo3?wQM`3r%_%+6koA zp%PQSgpzV|AI9bAFxGpf1p7#4JYBpr-F(VHV=UJBK-Jcqg;1{AI$o#CdV4YR|dSnpk9lXV7MYqh#==Dh{6e?@~wmZ8}f0jcC^< z(OJ54Fv+8zE|i)G!Yzz#{6*2DB9j`s9AmCl#bD-vqe2|a9x7XAz)BNU+I90K{nvM1 z*zuNO`RZW2T)B@1P5%r-d3S4z>%yR>?-3SHNaiotMWU*OBbw82a6c{%pVs8yf6LEe zZ2NDfZG9wT-RTde_1<{xx&#_Nw8G?(&zO#31~4|~32m{b?NUGjk|t&3^!xK^awmSc9^SfPDb6TZ36CBD%~u3vyUMr0}V! z$UN36C7x@Y*kOA9nA28GHdmI>%Nplt)JijIK5+p%e}g9L$#j!^-DOn#j04&Tp8J1& zrmRHvT{?a1H=69JCUAC#gPZCNVzuTR<78}%>c`teAM!VXhT|E09FtBGO`ej1_p0cg zKAQfNSw|*k$-}Xh{j5$O!RCWgS--8zdF|P9uuWh&O1pn1dV*Ize9eCFACp08x!5JE zAIBIY9WQ!(g8`LuE1+Me7ecC_(R9!F30n2f>DoOPL8UQ*Mw+-lcfKJTscj54io$Mr zya77(42TTH4}i^hOKd6NXhu!%iG;I*ap>}i}6O+#W2WkG#_yLFPGD3ga_dho|`DbAF^)j@0&s<_;!D;oJ^WT%lh;10Iy(#hxUQRF06-n(!EmC;#SNY)U0< zF%$SXnQH7nU?eEAi}>7VaoDpl9yRWj;Z`yo5)Z2IN8@ezjAz%F-LsZKxMmKl`jY~K zURF3?X_CM~EaPm&i;30YHfXF*BYRBr;k@`%T$7~>(+tkA4FY>>+QX+r(IblPFI~@8 z`KjUk=@P6*Vmdqa{4{Fv){*29N-occr)y@rLE9~$Lk~aD5%rB2eNBS*xG)EI+%KXV z7Zh{8qm`-Nzydr~xPz4I9H>q6h{HzJ4^-A@I_DEe$*5d&QuxdhdGYD&=^5Sh`YwIg zG|Uy#R=k1tiw=^Xc>=Ha(Rcb|nc#uSS;n0zcgDW&7WB#0k#y!+2lnlpXSGhMH*wM9 zOuXb_0fT$X&>`^&`I5E?mz7hxtG$bOMSF7Nq^d~$^Ve9snb0vVx#Xw)2sSB*VuM#M zcu5rF_NqXu)r=aImd+tQE}EfU$5jH9ZC==jbN!Y)b=-dZh1wVPi_ zm{tZFFB=b}L>jetZxXo32CjM5K}GTqZSPj(cM9IR*%g&=^=3E*?5l&{@fsLBT$g&S zi^1D^$)e=EW8_e}G|sC&1o^ApqW=m*bX#3do3>x2(|p3<3uBMDW4geRx96o!ou&t; z+vAZP*RgT(Pf_joF4E$V%0Hdh#zLJX-Fv8*N#-N*`o994Wg*Gl_z=i@UtfgbBivy3 zv}jzIlZ#))-XzV#)mWbmM?ouoJ^OQqB43a&9v)O4B~0vbeEmj__ho;8c5?x1o%4vg^{ilj zKX*i>vI5N)T;lgy zo8fwX8_KwxWt=9;vE@y>so5Y$dIyHMe{Od{Ph~Ct>SZzA-;$1UmD%P7USEkvh83^8 zb{_g~(8Bzp3(Wf&(ah(ySK;H%6jT=Sn=?`)=n9FI;5s{>s?QT+*OZSWuIntY(0(se z4Ozmvxt*-Z;Bq>@UJg}yR9JTAF?#s+bAIOgE1*5>8}B|$XodV!F8EOi8=;_pPu&-h zBEjRbx8WlFB{u^yy2QxWrV0>26igf$Ni4r@WL}wMk?9(zU^!PwmpY{r&wza-Qsy>u z@84-0x|NAf_<jzsTd|Vb~*jXr3}HnW+dc0Di?~=6d*D8Wo-iPj)>emjlJvupeLP zv-KvFIDZw{m>iWHk@O!8i3@@QaOO zRkv+~iL3@xTR8KFXP+WNT~V~leG(ZS@QJs{`bY{N?8ZxSm%#qR2u5-5a612)8!T`b z4db)Q=(HhAzVmx0t*o2}VbON%BmOtJQyBo!sTxGt`XZ5+I!AWKhtaIquVmW2NDMT| zWX&thaMHzzuv?got?nJ*j~sWv<1fa8(w_C?+TOYR;l>Jxe4I&pmud44uAhLGk>l7S zU;e_<^1U?ZhbHT%3{>g0Jgk47O+VW-!a9E)oGbj7V@j={PU$xvG_)8yw*oqSjsctW zjqIcA$+Ys3h|E)+2(1BEX_u$)`-&dJN2y=504+&<2xAcWRb*gNDD@-BQ>P?XrUzP)^AOAz>Tma@ySfhJAZb|hbj1cbUZ)!z@Fw^9tAR2mU4El^O@hT zRKUToh!LCejhm39$(|5=3|9`RVXL?+dHCNo^7P?=x#5aD$_!V9o~GlZ!!8^iRJV~k z(d(dd>MR&Ky$t_cR|RVFyT@-G^2qcS3@mCC;{tXPvfG!*rWbr0V<= zR99Mvoukf>bwO)begAv-H_C&S&npEtjR^W);0VpCp23cfHYdZ^RFJ8(gPXnoJH2wy z5}qk6<%%xdrk4a@(O%t`kg8mQkIP?TdSD-fPngU{zmX>0F}-wkeI_(6pT*Ct$OBH} zD4$ZZ5L#g*d~5KaAx*th*1r=|D;K~V)6v-cp#xWsdrh~Ngrl{02>dAZq*p8#(+LOl zpgF0Y-W9z5>c^vTKuZ@`ooMWKuO=s~)!FK#W$c$$O<3I@CfH;}=pptDRNN&6mfB`8 zwpM^)r-li9)fMDx#%*FB`;Pm)<|TbsodShV?_mC-W@2|Xi`>?0A)-)k*gH7^if(0~ zzTG3bWsZ!+m63_C@IoGI^J6Km?aPzj2Wf_hqP{+AIa>_0e3fe3hb@A~$DXoXI+LN^Vm2uMS`NSR z!g1q31}NQFhSwe}rztUIWZ@!Up9g2tCv7TZ)b<3F`zyn{=!*j#Q$sY3OhiQoHDSKH zI;*aK96mc;f!O{Ta51hJdJe}i@4tEqdGN*jfm&%|y6+I&o4S#t8)UL-leO@5`g~mO zvPsx4m-0F`<9Y8bnJ|!^LT+5!#+LYvL*<(yu1Gr{`ocexc%}DPET_z_>xrkkPxLca z&78=;X>MG%%W<~WZ{`NIJguHLVzTx}Mnn7D3z6*?4E|K6c!Q zd~o}<0rsCAiN~Y}ir(Bnoe{(M_JXT)cYPLE+1Il+3NP@;yKu;rR^*3{H}bYe(jd(w z3dP+@VNIPr(fX#z9=rd8p56G3woM#G3f`DO!1M8-w&5tkgApKGLg7dJU2d54BXsO; zV12J%p}}{z@j?qIy?5pif2XzsT`M+I?v@aX$`O!jl{?wsIwi1nD#lgH zQ|UrYo{qOajNCd?X3L-Nj8@rp%!==$%(7@ml%%BdsMB=iMH+dPIng;kKHZAE>|dT~!l8X5&8;=0fM zyAgz#%)3Oo0Te zS(B|k<48j3S{xs+0+^>g!do?K}x%#-E4rT->ZL zgTXNRU?{um+ea$DZ6xfoJjIL|IK+>Bx)H{N$g}STqCq}lBW!73hi}alA)&MgudVgu zCG&mp_rPWn{##%m>7T>anFsMdM|tu!R0;ol^oB9{O=Q=+4ibL&3QS9y4H`6aw6IuHwuQ=w?5pI7{Ix#-lP9NDKwc1{cJ6qjYwx@v< z-8aVDyKK>PrXMNYFido{bR5)0JZArjpXB11AQCMSI6&puDDK`(FMWtbFEv@by*!^% zw^(c#Ih?z-;wahBJ3v3T+JIY&61}`&JHCmmfsrnr;8Ab@vi?lPVx<$9JJgDP8^SO? zT`2XuR0;*@!kz2#GvYq^6^u7+W6Ga@C*pys{QA?c$o)A>(YkaS)6*M(t0pUBs98Dn zotnh{C>f0j+$oyAH4OXsuV{EXgdPzX%{DV@&F@dt)Chh(fF-oe+_N zd;ELQ`x%g0fq``7;}^`V%cmJW86?1tBQH}HvKe>*GBShlsZJQ(RFzJZjAU?;(82Ss z_cacM{(^N9rg-gQF)m)I14?i0xZCgU!Qu>MqWohVzWMl{D8zddCP~V3jr2V}^6sSj zNCwQCH-Vp-I1hV2m|;qmKPv9`fgdH0$o$vK@v3|rXsFC)a-7ED)xIHWV^PN`k8(uC zF>}nTt8!qha{wsHRl&AiPkJ)Pl{?#ehoPmh}hrEwe>=) z`RsWR9n(Vp3??zPy@1upCy9&cWSCmcz&4M1-mt2Kc=d%q#HDKTR^Fh&m<4cR z;%j(6qm0UZi6ow@uTV`RX_A1B?6xsegme24$*$T?H~UT`vp0K^!}6N+w(&K3JUowE z@+u!jHTF;=BUAL5yN@Wm3F2?byd#bMw?tM8N0KE245?l|3LbIx_(D9ES}&`iKTBH4 z=*%mkt~nui>3s%hY%?AaKqn{C_G8g)5{Z`^~{t8@f zoC{UI&SKyi6TDz1!muL?Fyy=ryDOp_636Qj{%lDF2U@?Xjjp{TMi+&-(x=AbNY!&s2nuWBr5+w7K6S1{^Q;z= zx|(6*Cq(1x%O%iTb{S^)6_G7Z{m@#nfPNgg39nAN0WBTRss4f6bnOcXa8x}&dutmg zz6d69fBK2&{12)e9>R~3%*S}Qg*Z;QYi2&&$hZBBr1G0oQSZT1=#ig?y59s|u7eV) z&o9FE<^Ig^vxPWsYb5@gZU7~hGQiT_oZcbXr0jS){%9y7FW#IYwp~5c{m*Y=kT{$7 zm>vOB;^biGteGInoK9lah5)bKLUO*X0{?X{iO9)_U3anq+eQN&jCQ5pg>SJ~LLhVH z!$kV@!bvh=nI)}1ITns~o}!T@PXvZgBPRY}SaU_Nx&_mmuM~)lKXUP_U3~^2A%!huN*K4%4UbiN!7ZxD)&e{`|ZY z-aYFNx4U&Nb4VwV_AK)yojP(fvULG*EzxH;q;)g7m5zLdsE+m|Z9``FTe52YIQUs& zk0GVQP_$JJR+O2u`|d9wStpFy4c4EbdRPg`H)&b$TH8Fp;kWx{wIM|-pTfT!gJ{d5Sg`j273cq^ki%NJkwUKSV$ z067=);B~_IIEmIiY{1SSd>_MmTHIt~7 z2k|%h>E>j6T2PgW2CipmLaQTwp7acdzDyA1%}m0_8{dGqWHRGm-Gkz9UCC~*v1Co4 zJX#Bx(Fu*pNRt=C;8kmsJrayfg^I#{=Q>p|%_FrV^@-WuXkN|rIGjH&&+Xke3J0bG z?G?^`#{UHWd^o~r=@xKui%LN+{4z$Qd_tz=8FPI93##5rKNd+ti|_OsjQuohi>HcFp=QZGk$1nIdmAdQ|#}Uvma~S00-9?x%0zZbVKsKS4ZkeeFK^A+l zrdb?4b~RC%4>tvlV;XnYM;7!#k}%OV8X6WPiL35 zP<=}&BeTdTTOyQ763%^{6iGuxY44>%G>j&{^ZO5;*LgkXdG7nVKA-oyc2c4clk{e3 z&OdK3e*BfZoVXY+*?Gdj%n5L5;waAL^?5nk7VbZN9 zoKB_-=h0e7y!KZz;k&-jxK-J-XJ#E9_u4@UBrX%~w+|W#d+yO8 zFoD51j!ickj`tQ$CC=xC-9uFq{%(3eHH7p2$*>-7x?CeOuF)7b9h*cOvW|&lI&X3} zW8_Hlg;ngpUt^j-R3!RPSs*aj@?f=m7B%;LBk*G#*-t{ZXwJ?&qCf8#`X=wfW3w;w zBbBEU!zF6`i7%~CrBy>@gI!s1d2!s>Zw$6?D8$T&gz4>pu%K1Q7<5Adx@7~JvEqoh1{v{rxohoandtItn<^xr{8siJz*sN zC0TgBFOwcrauDf?e?;Y9#kgXe4F5j%im=O3B_>b(z)|p7X_Eg{x;sw59C zo3pZe7qT?3zyxDyZHlFwpc<~yF9(G&zdXAE)nsh8}=zwIDh;6W`u+{NWf zX0fq*4r678Ko4l~V&%7#(lx_OQM=0x-(RUEOGfLk-!{gAM0lLQRMTcFrVq!Z$%je8 zxdiZGGnu;%?bs_WB1ic*7pHIlN-t?E6G_sXWBY=Yiyh zwFO+*_MS{!Tt}iV*ztGf-a^R~bvR;hj4_XqC4))hF>bCYv^lq6C_9|Kinhi3Iwkn+ z*M4Sr=^^%waL4<1`WieFdJX+|ST5Bl4LC6e?Cv#WYcgE;60_k#9=3&gMoIHuKAp#e zfG3#!+8!@Tp8-Ra26AKm1)N>~oLKaAfnMnf=EWX=PF-Pmy7l68dEV?Hzee- z)7O&_^RcMZ(?bIy@wI%)B z31Mb5xP;;3YSWn?H9K(Wy3;sswKU3V6RPGL4bI0Ds7_!9eLYi}I0Z9QQo4FR^bJlt_c*<^T3*JjFh0mB9ct1 zNu?{vWBjk%1ZpiNVfngAZ1xFZ?+`u(>Rnt=>qs-C{C&y1JQ7cTlng>s*>c!;z7YB5 z=XCIyGtGTdgHn4vNS8z*WgTw_a{wnYq^$sHlRhvbJM3_I;XnHOLpR)!TnjQ=W%*Z8 zd!V?|l3%{x0n64a(TGFRY{RiwQDJAZ$na<>4o)|Mg_*5byJ<8mTDu*`{_-V0<6pw$ z8xzSUg+a2Utdl(cz)`C&gTy8GD9-<-!Q1~5KF;HG=KB3&{(FifYcK0gdgglY_y35K zypzGOzIG0;X`scw{O>Pox4|Fp>PVws*E?>v`EtzV-{AhmrI;}(3S;vF3HxFuyw7T9 z#_xPVtY>HPc1xGx`!Uy1E%_XMF?0~~j>__@2lo&omoez%Ex|wBHIM&RR1a%Lk0xK4 zfAArED!W;dA})JN*4FtmpKdqdi0LZ4^zYR;ui=cqAsB{??*+VRScleUywNje7Y0<` zp&rdIv0il#F23)`$IaSIH-3GGp20=r)tWK3`>do{(a8X6tbZOqew)IlIE26lrx^Tx z^A6tB{7x;F9LKU*iFmfNjjH@rfnEPfXi4WK5_#O1Z;hAcmlbv3vQ-6mr}83hwaiC@ zn|9Q6lP6!d*MMKDZcd&)8=?!orNO8pQ~5W)w(=h?Xt4{!P5F_nY3Q{wjn~y0Pkc-c zlBS8C{O4*OFE=2{-TgMcDuZ5blx&*6fFC?41#EaeR^^Bh-H;m6 zIQcrS@o)jJb0LAR)cHb9GzluGBlErRFgt0YD!<3jfDTzP*jhFpN30%!%IqxK)u;$R zG6MMn=Nq^^k}<6AmPxgR#?PqhqouS+H5kTyH-d~chEVU_L1K>XVgs%U4qUByaO`n^owi!ZR=x(q-JQGK~F%h~n8-x_8Cb{`88%t8xq13Tw zWVhQrny{xHx44E=rmuu|`~8!)T)9E!|23mAx+x@H=NMTYJr(v(8%b9z7MN(i9-v9y zI}|nb(73R8NFOl^4Lsiq@6aZy@l77M&5x+$_j>rCp+JY_HNgQ}SANFOd(xU#Vq>+_ z8r3G{2xo?9(0g^Blpb(~Nq+H+e~S!g?QJGkZBl9f&6!{;qem_bI$@rf3CulTL+h1( z32yi_)Kg2vdV}NifpN3QcE33Lp;_o786APw|LsPDz5XD!(T<&z@rM7sKrp1C38f&ejVM)S|^czNboE3SwwkNCM2MlVdiGyhC1` zT0xAVj)uF=IPrO#o#enM-wd=&>2$#JE``dJR;WAIV(Ot;-PJliIS-S2- zEK#^(gHL|wbNhbD3LRb%mC89tRc?e4u-Sr*=Zxr_++xn^ku~~+s1mb^*PPma!d!4? z5QfNP!2DcAoE}m@?Y8p7qE(5=jr_qGs&C<9xA&2X+;vQYO%mN-8%|xwMiOeQ$d(P2 z(0sGQbf%Lk8dW4yh4ge%S5r&#H=dx9{qO0+m#)}&S{rtYWN`L~dH8J6WWGSv6%Svz zPF)j)etX3g^2Irpr0IBrq5Bfr-l;+&O?=Qv$jPcSrE}clHcFdrQr&F@xZzZ?bA)RnldK_~hY^=aPbx0t3a%iq^_0htU1qSP`G){poEflaYg^^`qq)$d`f zujqh8%XvI?ScSCPmQZ)C34Fh%7~R87F{+()3hPWI=^AN#DgqXD`o?18lsAY{N4rcL-k--V8%cKk!`?=wBR@ntEl zP&$+!@`dI%fp|x!l@6_S;4&t?gF~)`X!lc}r{ltDvmS$LuS1B* z<7c?VMc_n;)JgeBIo2)j0&EeYHmr)3^@2y5uPqpgCv=`F!1sLSrTSR zMqP{n<(V18uLo)N)nm99Tj-jT75tirYxLgn-So_?D4PDEo?d<{IJnqRY>ECudg4|B zJF3J0W*+V#%X?RYsjCrNo#=(r$3)^@aXY;DW(~sUW0>sHMS?B=kYSHbaAGa#bi%}b zs-ky@Ss`6PZ|9Y90pmk-lp+;XTPGiYnP64)nLkS(nqxORdJ z4()WNACnK`lzk(>Oy?DjTXcysBRAAu6ko#cyfG2>)LS#F0z*ia+Z{PLgm|8U{JP}DrNcNPL);ke77-rOMb%FiACf*^9FKu_R%3DH3-`zk8{4i#kNqxn;=bhe?pu!aVXzzlcoXgVA^3DS91MqjSvH)B3zL zDsI=wjoXrhmR*U&`rdX(kmTT}iv<7H>;)5es|S_Gv@jyDWtV4-z`stZ7&6LF*da~= z-5cq6AYnOX+V3K#mf51T_h`Odc?F3P?(QMJA~=86w)UBVI)u0{C30&G`Jm`(W=TvS z`+j#BBWczRB@ZolK6^2C?b-}&v%2w}>;#DXa*>>0T|qave-<+7qv1xvEv~`sAc?qQ z!a1HA$$VNbhY3H=oknGb&ON`r3$MTh~X!34ZNHE2*hoA5fTd7EC~hZx%URT z-zakJYkZ+}SOoTd(W0wOj*!euM>N@$!BtN@OK&8ff;qW{?B6~~SiR^r+4z%i<}^y= z|C$Q9u>BZ0w}@T}j1$hE$&m3Y5Z_exat&I|81nNfY0f>!b*|Y3;|_Py9B+Mid(9Xd ztfldN_aa_?qc5J39Se3}1g??lLUfV&LhJv8a`RMf;uxK0T!`Bio8B)GWaWh6xWrZx zqYD6^EN`bjDw0KG{>0F6Z%ybu<5Vi9dl4^o?-a5N?!;$i3o#DrCr>mP_`LcvHfG=C z-c&Z%B>%QXAuLbj7q_9O(jR*I=t^kr_(ozs@bL3hB;-uLz`V7SVdW!@FyOuu-tam@ z9?yP;z7Rp{j9c)^j$&?;OCbhz<#1wd!>P3}f47);8_&;iK+i=FITfg);cBL&q^*{I zTTo4tKQ+*V=GAoSpElxlGC|-_Tl0V4+#?}ECMfP-KDoZf9{+8bM$W0Xk;W${a7A|? zW^B=>&bb0R^pz>Td@z>v9rFi9cd)SeuLa|IT{ydG*^$|wUNA?+&(lXSKdF? z&Ca#_%nhF$4V50l;O7D_aG5#`Px|k&VL%$X_DiFQ+<6jygcWA}_iCnpj6mgLdlFo> zpY52x9;Ma?(*{okvc{_z*Q*}IudTiG?H)?oq{bunX8`vz!}#X~Df!{s<`TP0}U8v$7v$QT=br2V`BN-Ud=!;DYU zNX@;}_-;1!Gb_Oxg;p54@((E*6Uywf9>GUeo8#WAiBwWij+Kr(NRD6aqebnnX@Bip zNZMRLTu>LII|s$e>O0}X2J!ubDqX01yrX%!gHmapGnXGsk;a@K@d zbBg(<^#<%dwV4=QJDY}mI|46-UE03SQ~9~c5lr>zJ)~ydJ51Ns$Mup*)THP$26RXA z4_IZi;v`^XKt6d^bOBcxM&U|pD>@H^d%!XmJbFL}e_qdoO+I;0RpyLVUrsTqZXC{! zRpYmOEn$aBG;zV;3wUBLoGp~5VpJ${#{R)%LhdM-T6hFaYf6dpr{{F;Z{eJMu~alC zJDo0jB{=Cuw=hpCS|RaJ7|!>fMfU$&f_i`0BiU!( z=_LH61~fI~la#RU{P`3w@cW$#R}%_hb^IM}^kjcBp*9M7{{mK8w9>Mz z3WPdcfZc0~z}UiE^f-AvzR2*QdY8;uJ?B#DIBEttc}L(QlKG@Vxd?*(G=XTAI0SW% z#^Fo1kSx99#5-^<*>YOM{u!7GZT}{~gbxY$YWZ`%`N(#dJ@q6k-KGJJKhF@gJTb74 z3+8$fUUEA(P2e+vZL!PDnS>16NwStHu)__zVfsQ%A%hWx>vV2Xli*>n#8{8m>kQH3 z0ylQohD`c!Yb*?e27zSeTYk3bGIC1VlbLC^lV+GJv(E4Kz*yKvS2b#4uKIb>d2I>( zlA=xbWu}lasRQt3PCI??Hj+1u^TU%%l51x_%_ff5qfy-S8l4#tM)S25>BkRt_`Yf{ z9W&5|75zGF&OT3C`d}BBrWvA|s=%KLU(d+M55vsze9ZV6LpHrVj9T*3$=V6WxIwXM zdgYQWLoPXUHXl_<^V|uTrm_lB9L4CS4_DFiQye)SH=H{5YEFRZ}+og6NhT1mfLyFr6r`lIvON__W3@RS#K!hhBu z$jKmckl4EzlVq#tr}2SAdii1M@G^%O?0LbMBpiUaW#ceGx17eVKSYi`+QNJ8MucB( zc>A+Fyed%PyH+hj^$*_QlcWTz#`hD0pXHF`$zb%aWw_#+3KKSbN9~6FLZ;@(dn~jW zi?`l0P_^C@W}M5zm;V+J#z&KG_G_VbPc_(|q5CkqdlUI*WJJEr9|>P;Y)IJJ&G7Y` z9NYAE6pn1ZMfuQLawc*je{^UfUpG4g&5rky$)jRvzsnQhce)l!+Wkn6_b56>!5n6V zY{EsA54e_V)zIsC6jEGALV}e#FYj~(6ZiRo_ZKhGeB(NNtD8ajIht&YsXB-}7Gmje zYtcHpBPjMzlRnQ6hU;G*lAOI~&|~u`Xq#ijURxGUB&PyRnm!fD^fU;_>STN_?jvN? zLvpVE5YC(bgDP9?Vy;@(kQcjK=*4Ru;M$joPbVltrCJ=lv{uM3zvv-zB{U%7w;EZV zcS7VpwyXBao7+TGj?7oVOPj?vmEbs++F!~oMv(Q41USqs${)?)v zDWhK;Uc=h_y`obwJX+LilcS4Ii_U*dAh~|x+_~U3(jfGnrtV3=NGjZYzi+Ie2Ud~2 zrk8MFgBV|}e4A4ryB{*r;_%_M7P{|YImvYs!K)sNTCIAe+9R7Sv2gZ7GJkv^eh;GL zvM7oq53%zFagqw(goG2j_GK)#>Xfkj@@?8?3vVw1iZFE)BXOo}ShS;@iG5!TFW zR}a`F@B}sOBS}#@N19nB)?aal&HF%saXn{0o^6uCqL5E;&Yl#Hn^qSfns;P>bmg0ozXOcFZ(K3^;#^1dk80U|;Q8ZiXm{@~EeN;8`e|j*;#&tA zm%_-^#;de)D2*DAyoF-JW2orbc-$#_AA9}AleyRI(7oy--JMcQ{2dxF>g@`8ac&Li z?BltBb82wQ+aJ$eT*&>4wWaajY%$u2z~hr{@N1qfo;N=ZYYk4a4-DL?i+L#i4H%@~ ze+FW!|7`3H3n$cg6uR6kqmTO%QAN0C=C+nHoa$7zS>{ zCt1u_(SE!YF6iJ_9mKGSN_5T zENM0jU5lZQ>~pDH&lcY2-(`H+Tg=2rC*tUU1@y@FWU|D~60-TH_-<+nmF|i|xzR^( z%poO^l#`@h_oJ};5pwCx-{_!o3|E!3yf$Xida~`JB|2WqBGxf+)Otr9DB2CH)jD>a zE|*CV5F9S>ar8S}n==x1k3OKrT0_Df$z5PSs?k3UYjAgN6r`7*CPO8)?D5wT*i%(T zx7wd2p;_Hb)@Bnr^+7U3XRJdzp_{yNo)YP?YzO&m_N*|3BSwu3{+IEA`(62yTVa#O z92hdDt-W=yC}}y~?n^@hZa-Z_5%u>xwTyIbZif$M^5Nru6+`@ zap(iR@=FVUSv@9<|8LU%V-l>%8IH#XzSENl+A#Z`G25(_L&~f65f!tm8F$gY=mXQj!{qjUGUR17Gi}i^UmrVa`B)U z%zdRrTx zd6ngjVt?Ifwa;VfG3_LCsDB^%Io<@x8b8_}yAlkgl<{6h801aQ2YzobR$sUd+uz5K z?1vpRGDp~LzTZkl_z3aqv?CZ9+XM4YiVn@^FnG=im9r-5&|7f70$qT;m@x^&@i)O@;w$^~Vy zb~68gzV|`YIIaT$Q~tsI=4ePgvlc_1$AH2JDVV(E6!Gq_pnkqAICBSx%rh@spuC!W zH~uN;znux0&fmea^egTKff1}z!ASIGV^sA_YE!%kS<^IxZ?bs*?hWd(W+YBgH3jLT zi{O!IKKW~E$;V49We2onaMFSVGNvMxTm0iXN(mkAC_PyWdy$HDiS1c~mlI8awW#|giWzxGyf;kr3A=t~hcU3vtDrDJH-0S7Q#w}>@m zqxr{cR}h`HV=#dnpvm(z&^7Ej>KObcT>>xLW5@>T^$2^vL7DEkegxOgBg}!cUv&O1 zE6mieBTb!|bW!25+IwdY5@=L{KbT4zKiIPIb@Mrcx2k;G3sqk4-D~WcA`bVyXP~2n z9^{4O!|}xhxG1y@{tNO3x-t^VP8M*3se<0!W*YRVzr&`iad5jmlxTJz1`X$>@F@JU z=+&*$V08TgIr}vVY|P%_6nTLqBD~{oLLS1~`Srr*u?6qF0&`>0Eg++fY43uactKRj zIcN>gC&#?eai$pz>)FpFMVpDX{b?tyL1WmwN+;+LIkA_TgGt(V3t=beN|p>pahBWa zaO|aWS}yF_4=pQ!Cq|p`wEJmvlu?9qHA}A3^abG-mExki_ORPt5zZgl5B`BWu+%J` zzB{=PHM~o3x6v7r(wxUBy@;j`wvEInb5bH48L4(7}m zpc4NULQLBxJaZxnR(529zQGAB6gbi;^G*`ijxq47U6O4R3&4v~kLdkP%50*p3LIS) z!t4>U8|s3?J$}+Y8Wko7-&zPN8f?Jw3F_E+_C7p$D+T+$mokN8X5*P_Uogv3g3p^f zik@2%PZz2xUD8`J8Yee>;{4Am5|Q72%1+b*zHJ-Tp=0=MWxA*}B*G*U9lqTmL&(-5 zwMsfGcrJY)Je0@vToPlFS_LEbFXXj!qOfb#WPbcTasK!m4>(gkoAj88vl{I&--;l3b&RDxA8x4~MXKF_U zep3HV-YuRfbooDVs;>(;`&vboHMmVQRwj`o^()-M2kv;IEdqAOsKS*i6Zo@9x%j8x zF_-?JfYk`oC2tp$;*UEnSf0C;?a`e=7v0XsQzzP~-*7pRoAgKSQD`Jicv4Otr<;P5 zx{x89PpPzI8cOaC}bc*~5Z`;K-a%nXJdrWsV3X=m)B>@e@S1F_wsNVa?v zV|Qnp;GN`t?rvoT_3?0kRqjjK>KCD)&}K@~uMfwNj9jek3qsTA6Ev*3n=$sSL?3F% zzno#h&8a&_&p9PP<@ZnbkvMGSk z4W!O|rS8}3@#*nzjF^EYQPlK7F7PP+mR7}^(>6h%l)wl#^kTgyZxdX$73AANGyK#v z20z#o(0=P%F#ml!b9l2RZc&{E8&t1>fm$LooNeYNIVTchPq{Ghy7f>W`Y^>S@sQeF;Ri~(O1Cn=5c6}tHA(xp~y(KX4RQkasV6gU@Nf0-BySM1}IbX<8 z5qhn{BT#!-L5?*FS)Z3`tdzbgjJ6BHNFnbe4gf9fHPk05btK6aGnkFAV>L~6{oR67mjgYyW zQpMs_*jul}viFkd)VO?_QD4VA)Sb^o{n$&NZ1Thf2?qsE>kqPc)HWR29Rp&UPSRS( z!+5d73LktJhuR7}-K)I>W0tO``vaVCOq&O0Z8idLmu)1+y&9B$M3C6a^=NfWogF;) zKvY@05C(4DgM2#)PIFM1HILd(0&jMa+s`(@j{|MkUUG^Qyptnk@fYdtm0nPgS_2Pv zbs-}-s6(uNk%cFw*rdp*v7_qmkcG9;WLwH;e!eic_f@$-XZd(CRSiwp*sF}5_pVak zvxf!Gq796QF~FHyCy+Z+{P1ddHIAD;AKkNsd)djUByy9$Qt*06CT>+DMdniQ`C2fO zZ=Qj_y(u5teTAv&HQ`TvO<{-E*OR3_8O+)DrPL}dhXncwv-lIy=%25|y55xJ>^=-| zYuV|+7Ov!BeC>mc27@lf%ThrE|-$veY1dj0STx}?DXe(ny0 z|Nd&wC6iE;GcN#-jq>Nlvj-Wu>fbhcv6&EVo=?XlT&FAZR#CCK7Cb-y7}+yn9RC+5 z)1GWEOt~V02W$*i=W_yHkDf+$^?nf?%(L-D_z+!p&mOJB3+W!U2D+wAn%-D)gD9Bj zqT_+T)S=f5&xRbNE5yA~r!N{V6{#|q=>eI53KJsLSrhgs22X>I$P+90Q`^qg8P zCvmwH_1_5IfDBoDqOlrf*GGbb&?_ z5o;%I>-W)+eTPMxoKBHV~5 zNNjmTH~uQb*KVuPk$S+zk|hxT>?k(uauzyyGFbjd4kjrLXN#ky*du05Oq_omlloW4 z)E`9@n=P=WPl}-ShBX#kJC9Wk)>k7}rOvncQyvJFTW$`2B?&2DO$yOE7LJwb) z<%dhl(u0F~;4!5Imkt->^l!|hV}|MBg@6#6fAclH*l?4XKK(JhAT0}XS|ixJnbSaT z^?01Kt&8@lUIew#AozBpHttD}i>PgC*NZ40Uq zpg?3b!{Fe(yNs)u4#;R9s$Mr@g0MeJLP>o;jJoE`zqUq^L&|CaJznK0o|6uq%$Bpy357GARN(POXBbMSVAZ!d>|!LErKMoKCl5Ri7qaX9sEV+fH(n>Z655 z8vj+XnsP?VVO*~*p4O`%t8dg$=dIQ_^;?KHv+#qM_Oy{UPJ0JQ-Zmt!TAU0Hz2Gw$ifgs~h+>x< z8&Z(RRa{*P9=G>F>7G1XN>-44+ihsd^gB!my(JnJ-Hh4Wd&%zF0g_$JWB9XQ%-N4~ zncmmUv{>gPxV+p`@ar zCckVY(c2n@|E0R&rW7g2__2pydul4m?pV$$z3ZS;l~c(A<^%o52@dhX2~hRUikxwo zN(`-)IqS%Sq;2juI4Yi-xE7f?zz^ z$`^hwAoCvmAeH|)lcP(*xJKDqbb@gK*(NVb<;oNB&N(-jGq;?2d&=-9Of697)kA7? zCX=K%KZcJguDtQOenvHa2YKPWnY1O%VBe1&&LLydT{ngVtS#kx$CY8q?ETdFW*$tsY$bF`R^pA1T6DzH)z}eM&Ckh7zy$RuK4fho z>lz?Wy30Z^ZS@WGmH9(ewnt(487u6bC3LuVCev3V573E@Ir!{p61a-(KvnrpJh%A? z=RQ0fOyTbNAetZmD*c&K$+=Ra+?$5>+?sU zRE6+veF}v3=?n3MS0?`1ca$b49Y*(G_2^_#$|!GmLuF-3(c7S%-25wzA~PRu#tK8h zRT)R$h?-H6vbaGh4HtBl;W;ZF4W6GzS@V?;RC}4Ky~x9T)0be)@5i|N$OWAD>LX{D zEXgZ&Z$gjE1a$j*j~}+rnh%*`MsG=JQ+hBI8WYq&B_*0K+Lgt$S;dhmJdJsc8~Fa| z7m3ZIeoS=@<;?>f(OWAHUkD7-NFjrNwPOI!s;;KWpW<;%^ksbbRfGTIpbI6!^RSV- z%AJ#x;WwvkjF=M!jX%h5Rur}PXI!Tit ziEduAnHnU?kgksPWZh5zT1ksTW5fg^{@*;oBlwkc8{eV(H3pd5HO5T$xNz{Q--L(y zpU|OyIgoI8IF9wxW`B&8Vx5ep@Wtd1o#Z+Wp55O_a_3wp8G|or%&&9M8Bs*WM%^Zi zY%DqP(}nD~P=GJHFMyO;0QYC7TWv#C1={9SqIhZvy>dGq$kBFkDVMUBk1RxgF=0j^ zurZ!4HH4dSKGZLD8ExTXN!wi|2-J*%DH@aU!ICa&vT!lnU1EnHCZFPVZFj+W&daG! z|5!9{b$~h6*`o26W}x0Cb@I^tGfaEcTbq-TMszy0L1K>rsNRm{Uv9ZB+&6eMIX@hy zRve=RX)(C??h|HjaW#}5^`X}jB)E4rOKYt@-K4=wl=!7i#qjUDh|ErK#O!M;;ltDz z2;964{)p|t5qoEm(c-pPp>Yx?oR>i#VP;=06H3Fje`h{#aI6;lyMS#AErw6IKq8f+ z$>EkYxXd${L}<#g;)2^Rho;fm5mQKnj~X^K+2U{wSCn=iwEGC}&z#SIO_+Ql@a$j+`@LVo1 z4ORM(JMfnIbhqF*Q_%WFP?1yy*L$=G4sO3$`kNG!X+X-L!R5Ybt%pFY9&R>U(@=kDq