From 493fd25759799ed9aaf7327de962cc72384ec8db Mon Sep 17 00:00:00 2001 From: Matias De lellis Date: Sat, 18 Apr 2020 11:22:54 -0300 Subject: [PATCH 01/16] Suggest similar person to rename them --- appinfo/routes.php | 6 +++ css/facerecognition.css | 3 +- css/files-tabview.css | 2 +- js/fr-dialogs.js | 62 ++++++++++++++++++++++- js/personal.js | 78 +++++++++++++++++++++++------ js/similar.js | 44 ++++++++++++++++ lib/Controller/PersonController.php | 34 ++++++++++++- lib/Db/FaceMapper.php | 45 +++++++++++++++++ templates/settings/personal.php | 1 + 9 files changed, 255 insertions(+), 20 deletions(-) create mode 100644 js/similar.js diff --git a/appinfo/routes.php b/appinfo/routes.php index 9c02d49b..740c97d6 100644 --- a/appinfo/routes.php +++ b/appinfo/routes.php @@ -7,6 +7,12 @@ 'url' => '/clusters', 'verb' => 'GET' ], + // Get a similar clusters by Id. + [ + 'name' => 'person#findSimilar', + 'url' => '/clusters/similar/{id}', + 'verb' => 'GET' + ], // Get all clusters filtered by Name. [ 'name' => 'person#findByName', diff --git a/css/facerecognition.css b/css/facerecognition.css index cb5ecddb..2f2eae94 100644 --- a/css/facerecognition.css +++ b/css/facerecognition.css @@ -39,7 +39,7 @@ * Rename dialog */ -#fr-dialog-content-input { +#fr-rename-dialog-content-input { width: 80%; margin: 6px; } @@ -47,6 +47,7 @@ .face-preview-dialog { background-color: rgba(210, 210, 210, .75); border-radius: 25px; + margin: 2px; height: 50px; width: 50px; } diff --git a/css/files-tabview.css b/css/files-tabview.css index 3d09bcf6..c85c5b8a 100644 --- a/css/files-tabview.css +++ b/css/files-tabview.css @@ -29,7 +29,7 @@ /* * Rename dialog */ -#fr-dialog-content-input { +#fr-rename-dialog-content-input { width: 80%; margin: 6px; } diff --git a/js/fr-dialogs.js b/js/fr-dialogs.js index d40196b7..6b14d48a 100644 --- a/js/fr-dialogs.js +++ b/js/fr-dialogs.js @@ -26,7 +26,7 @@ const FrDialogs = { rename: function (name, thumbUrl, callback) { return $.when(this._getMessageTemplate()).then(function ($tmpl) { - var dialogName = 'fr-dialog-content'; + var dialogName = 'fr-rename-dialog-content'; var dialogId = '#' + dialogName; var $dlg = $tmpl.octemplate({ dialog_name: dialogName, @@ -87,6 +87,66 @@ const FrDialogs = { input.select(); }); }, + suggestPersonName: function (name, faces, callback) { + return $.when(this._getMessageTemplate()).then(function ($tmpl) { + var dialogName = 'fr-suggest-dialog-content'; + var dialogId = '#' + dialogName; + var $dlg = $tmpl.octemplate({ + dialog_name: dialogName, + title: t('facerecognition', 'Suggestions'), + message: t('facerecognition', 'Is it {personName}? Or a different person?', {personName: name}), + type: 'none' + }); + + var div = $('
').attr('style', 'display:flex; align-items: center'); + for (var face of faces) { + var thumb = $(''); + div.append(thumb); + } + $dlg.append(div); + + $('body').append($dlg); + + // wrap callback in _.once(): + // only call callback once and not twice (button handler and close + // event) but call it for the close event, if ESC or the x is hit + if (callback !== undefined) { + callback = _.once(callback); + } + + var buttonlist = [{ + text: t('facerecognition', 'I don\'t know'), + click: function () { + if (callback !== undefined) { + $(dialogId).ocdialog('close'); + } + callback(false, false); + }, + defaultButton: false + },{ + text: t('facerecognition', 'Yes'), + click: function () { + if (callback !== undefined) { + $(dialogId).ocdialog('close'); + } + callback(true, false); + }, + defaultButton: true + }]; + + $(dialogId).ocdialog({ + closeOnEscape: true, + modal: true, + buttons: buttonlist, + close: function () { + // callback is already fired if Yes/No is clicked directly + if (callback !== undefined) { + callback(false, true); + } + } + }); + }); + }, _getMessageTemplate: function () { var defer = $.Deferred(); if (!this.$messageTemplate) { diff --git a/js/personal.js b/js/personal.js index 971756a4..6cd589fd 100644 --- a/js/personal.js +++ b/js/personal.js @@ -124,8 +124,9 @@ Persons.prototype = { /* * View. */ -var View = function (persons) { +var View = function (persons, similar) { this._persons = persons; + this._similar = similar; }; View.prototype = { @@ -156,6 +157,63 @@ View.prototype = { } }); }, + renamePerson: function (personId, personName, faceUrl) { + var self = this; + FrDialogs.rename( + personName, + faceUrl, + function(result, name) { + if (result === true && name) { + self._persons.renameCluster (personId, name).done(function() { + self._persons.unsetActive(); + self.renderContent(); + if (true) { + self._similar.loadSimilar(personId, name).done(function() { + if (self._similar.hasSuggestion()) { + self.suggestPerson(self._similar.getSuggestion(), name); + } else { + OC.Notification.showTemporary(t('facerecognition', 'There are no more suggestions from similar persons')); + } + }); + } + }).fail(function () { + OC.Notification.showTemporary(t('facerecognition', 'There was an error renaming this person')); + }); + } + } + ); + }, + suggestPerson: function (suggestion, personName) { + var self = this; + FrDialogs.suggestPersonName( + personName, + this._persons.getById(suggestion.id).faces, + function(accepted, close) { + if (accepted === true) { + self._persons.renameCluster (suggestion.id, personName).done(function() { + self._persons.unsetActive(); + self.renderContent(); + self._similar.loadSimilar(suggestion.id, personName).done(function() { + if (self._similar.hasSuggestion()) { + self.suggestPerson(self._similar.getSuggestion(), personName); + } else { + OC.Notification.showTemporary(t('facerecognition', 'There are no more suggestions from similar persons')); + } + }); + }).fail(function () { + OC.Notification.showTemporary(t('facerecognition', 'There was an error renaming this person')); + }); + } else if (close === false) { + self._similar.rejectSuggestion(suggestion); + if (self._similar.hasSuggestion()) { + self.suggestPerson(self._similar.getSuggestion(), personName); + } else { + OC.Notification.showTemporary(t('facerecognition', 'There are no more suggestions from similar persons')); + } + } + } + ); + }, renderContent: function () { this._persons.sortBySize(); var context = { @@ -224,20 +282,7 @@ View.prototype = { $('#facerecognition .icon-rename').click(function () { var id = $(this).parent().data('id'); var person = self._persons.getById(id); - FrDialogs.rename( - person.name, - person.faces[0]['thumb-url'], - function(result, value) { - if (result === true && value) { - self._persons.renameCluster (id, value).done(function () { - self._persons.unsetActive(); - self.renderContent(); - }).fail(function () { - OC.Notification.showTemporary(t('facerecognition', 'There was an error renaming this person')); - }); - } - } - ); + self.renamePerson(id, person.name, person.faces[0]['thumb-url']); }); $('#facerecognition #show-more-clusters').click(function () { @@ -260,8 +305,9 @@ View.prototype = { * Main app. */ var persons = new Persons(OC.generateUrl('/apps/facerecognition')); +var similar = new Similar(OC.generateUrl('/apps/facerecognition')); -var view = new View(persons); +var view = new View(persons, similar); view.renderContent(); diff --git a/js/similar.js b/js/similar.js new file mode 100644 index 00000000..f376de0e --- /dev/null +++ b/js/similar.js @@ -0,0 +1,44 @@ +var Similar = function (baseUrl) { + this._baseUrl = baseUrl; + this._similarClusters = []; + this._similarRejected = []; + this._similarName = undefined; +}; + +Similar.prototype = { + loadSimilar: function (clusterId, clusterName) { + if (this._similarName != clusterName) { + this._similarClusters = []; + this._similarRejected = []; + this._similarName = undefined; + } + var self = this; + var deferred = $.Deferred(); + $.get(this._baseUrl+'/clusters/similar/'+clusterId).done(function (similarClusters) { + self.concatNewClusters(similarClusters); + self._similarName = clusterName; + deferred.resolve(); + }).fail(function () { + deferred.reject(); + }); + return deferred.promise(); + }, + hasSuggestion: function () { + return (this._similarClusters.length > 0); + }, + getSuggestion: function () { + return this._similarClusters.shift(); + }, + rejectSuggestion: function (suggestion) { + return this._similarRejected.push(suggestion); + }, + concatNewClusters: function (newClusters) { + var self = this; + newClusters.forEach(function (newCluster) { + if ((self._similarClusters.find(function (oldCluster) { return newCluster.id === oldCluster.id;}) === undefined) && + (self._similarRejected.find(function (rejCluster) { return newCluster.id === rejCluster.id;}) === undefined)) { + self._similarClusters.push(newCluster); + } + }); + }, +}; \ No newline at end of file diff --git a/lib/Controller/PersonController.php b/lib/Controller/PersonController.php index f26664a4..423541fa 100644 --- a/lib/Controller/PersonController.php +++ b/lib/Controller/PersonController.php @@ -46,7 +46,6 @@ use OCA\FaceRecognition\Service\SettingsService; - class PersonController extends Controller { /** @var IRootFolder */ @@ -172,6 +171,39 @@ public function find(int $id) { return new DataResponse($resp); } + /** + * @NoAdminRequired + */ + public function findSimilar(int $id) { + if (!version_compare(phpversion('pdlib'), '1.0.2', '>=')) { + return new DataResponse(array()); + } + $sensitivity = $this->settingsService->getSensitivity(); + $modelId = $this->settingsService->getCurrentFaceModel(); + + $mainPerson = $this->personMapper->find($this->userId, $id); + $mainFace = $this->faceMapper->findRepresentativeFromPerson($this->userId, $id, $sensitivity, $modelId); + + $resp = array(); + $persons = $this->personMapper->findAll($this->userId, $modelId); + foreach ($persons as $cmpPerson) { + if ($mainPerson->getName() === $cmpPerson->getName()) + continue; + + $cmpFace = $this->faceMapper->findRepresentativeFromPerson($this->userId, $cmpPerson->getId(), $sensitivity, $modelId); + $distance = dlib_vector_length($mainFace->descriptor, $cmpFace->descriptor); + if ($distance < ($sensitivity + 0.1)) { + $similar = array(); + $similar['id'] = $cmpPerson->getId(); + $similar['name'] = $cmpPerson->getName(); + $similar['thumb-url'] = $this->getThumbUrl($cmpFace->getId()); + $resp[] = $similar; + } + } + + return new DataResponse($resp); + } + /** * @NoAdminRequired */ diff --git a/lib/Db/FaceMapper.php b/lib/Db/FaceMapper.php index 36b83c8e..a63d3a80 100644 --- a/lib/Db/FaceMapper.php +++ b/lib/Db/FaceMapper.php @@ -31,6 +31,8 @@ use OCP\AppFramework\Db\DoesNotExistException; use OCP\DB\QueryBuilder\IQueryBuilder; +use OCA\FaceRecognition\Helper\Euclidean; + class FaceMapper extends QBMapper { public function __construct(IDBConnection $db) { parent::__construct($db, 'facerecog_faces', '\OCA\FaceRecognition\Db\Face'); @@ -131,6 +133,49 @@ public function findFacesFromPerson(string $userId, int $personId, int $model, $ return $faces; } + public function findRepresentativeFromPerson(string $userId, int $personId, float $sensitivity, int $model) { + $qb = $this->db->getQueryBuilder(); + $qb->select('f.id', 'f.image', 'f.person', 'f.descriptor') + ->from($this->getTableName(), 'f') + ->innerJoin('f', 'facerecog_images' ,'i', $qb->expr()->eq('f.image', 'i.id')) + ->where($qb->expr()->eq('user', $qb->createNamedParameter($userId))) + ->andWhere($qb->expr()->eq('person', $qb->createNamedParameter($personId))) + ->andWhere($qb->expr()->eq('model', $qb->createNamedParameter($model))); + $faces = $this->findEntities($qb); + + $facesCount = array(); + $euclidean = new Euclidean(); + for ($i = 0, $face_count1 = count($faces); $i < $face_count1; $i++) { + $face1 = $faces[$i]; + for ($j = $i, $face_count2 = count($faces); $j < $face_count2; $j++) { + $face2 = $faces[$j]; + $distance = $euclidean->distance($face1->descriptor, $face2->descriptor); + if ($distance < $sensitivity) { + if (!array_key_exists($i, $facesCount)) { + $facesCount[$i] = 1; + } else { + $facesCount[$i] = $facesCount[$i]+1; + } + if (!array_key_exists($j, $facesCount)) { + $facesCount[$j] = 1; + } else { + $facesCount[$j] = $facesCount[$j]+1; + } + } + } + } + + $bestFaceId = -1; + $bestFaceCount = -1; + foreach ($facesCount as $faceId => $faceCount) { + if ($faceCount > $bestFaceCount) { + $bestFaceId = $faceId; + $bestFaceCount = $faceCount; + } + } + return $faces[$bestFaceId]; + } + public function getPersonOnFile(string $userId, int $personId, int $fileId, int $model): array { $qb = $this->db->getQueryBuilder(); $qb->select('f.id', 'left', 'right', 'top', 'bottom') diff --git a/templates/settings/personal.php b/templates/settings/personal.php index f4119ef4..39a55117 100644 --- a/templates/settings/personal.php +++ b/templates/settings/personal.php @@ -4,6 +4,7 @@ vendor_script('facerecognition', 'js/egg'); script('facerecognition', 'templates'); script('facerecognition', 'fr-dialogs'); +script('facerecognition', 'similar'); script('facerecognition', 'personal'); style('facerecognition', 'facerecognition'); ?> From 333d649b48ba0618f007bbead9b356263f08ab9c Mon Sep 17 00:00:00 2001 From: Matias De lellis Date: Sun, 19 Apr 2020 12:11:43 -0300 Subject: [PATCH 02/16] Add a "deviation" setting to control the suggestions. By default, the deviation is 0.0 and therefore is disabled. On the other hand it depends on dlib 1.0.2 to use dlib_vector_lenght() --- js/admin.js | 56 ++++++++++++++++++++++++++- lib/Controller/PersonController.php | 7 ++-- lib/Controller/SettingsController.php | 6 +++ lib/Db/FaceMapper.php | 8 ++-- lib/Service/SettingsService.php | 14 +++++++ templates/settings/admin.php | 15 +++++++ 6 files changed, 98 insertions(+), 8 deletions(-) diff --git a/js/admin.js b/js/admin.js index ff0f7912..fd007f9b 100644 --- a/js/admin.js +++ b/js/admin.js @@ -106,7 +106,6 @@ $(document).ready(function() { }); }); - /* * Sensitivity */ @@ -161,6 +160,60 @@ $(document).ready(function() { }); }); + /* + * Deviation + */ + function getDeviation() { + $.ajax({ + type: 'GET', + url: OC.generateUrl('apps/facerecognition/getappvalue'), + data: { + 'type': 'deviation', + }, + success: function (data) { + if (data.status === state.OK) { + var deviation = parseFloat(data.value); + $('#deviation-range').val(deviation); + $('#deviation-value').html(deviation); + } + } + }); + } + + $('#deviation-range').on('input', function() { + $('#deviation-value').html(this.value); + $('#restore-deviation').show(); + $('#save-deviation').show(); + }); + + $('#restore-deviation').on('click', function(event) { + event.preventDefault(); + getDeviation(); + + $('#restore-deviation').hide(); + $('#save-deviation').hide(); + }); + + $('#save-deviation').on('click', function(event) { + event.preventDefault(); + var deviation = $('#deviation-range').val().toString(); + $.ajax({ + type: 'POST', + url: OC.generateUrl('apps/facerecognition/setappvalue'), + data: { + 'type': 'deviation', + 'value': deviation + }, + success: function (data) { + if (data.status === state.SUCCESS) { + OC.Notification.showTemporary(t('facerecognition', 'The changes were saved.')); + $('#restore-deviation').hide(); + $('#save-deviation').hide(); + } + } + }); + }); + /* * Confidence */ @@ -263,6 +316,7 @@ $(document).ready(function() { */ getImageArea(); getSensitivity(); + getDeviation(); getMinConfidence(); getNotGrouped(); diff --git a/lib/Controller/PersonController.php b/lib/Controller/PersonController.php index 423541fa..952a9565 100644 --- a/lib/Controller/PersonController.php +++ b/lib/Controller/PersonController.php @@ -175,9 +175,10 @@ public function find(int $id) { * @NoAdminRequired */ public function findSimilar(int $id) { - if (!version_compare(phpversion('pdlib'), '1.0.2', '>=')) { + $deviation = $this->settingsService->getDeviation(); + if (!version_compare(phpversion('pdlib'), '1.0.2', '>=') || ($deviation === 0.0)) return new DataResponse(array()); - } + $sensitivity = $this->settingsService->getSensitivity(); $modelId = $this->settingsService->getCurrentFaceModel(); @@ -192,7 +193,7 @@ public function findSimilar(int $id) { $cmpFace = $this->faceMapper->findRepresentativeFromPerson($this->userId, $cmpPerson->getId(), $sensitivity, $modelId); $distance = dlib_vector_length($mainFace->descriptor, $cmpFace->descriptor); - if ($distance < ($sensitivity + 0.1)) { + if ($distance < ($sensitivity + $deviation)) { $similar = array(); $similar['id'] = $cmpPerson->getId(); $similar['name'] = $cmpPerson->getName(); diff --git a/lib/Controller/SettingsController.php b/lib/Controller/SettingsController.php index 5a635cc7..abd35389 100644 --- a/lib/Controller/SettingsController.php +++ b/lib/Controller/SettingsController.php @@ -188,6 +188,9 @@ public function setAppValue($type, $value) { $this->settingsService->setNeedRecreateClusters(true, $user->getUID()); }); break; + case SettingsService::DEVIATION_KEY: + $this->settingsService->setDeviation($value); + break; case SettingsService::MINIMUM_CONFIDENCE_KEY: $this->settingsService->setMinimumConfidence($value); $this->userManager->callForSeenUsers(function(IUser $user) { @@ -227,6 +230,9 @@ public function getAppValue($type) { case SettingsService::SENSITIVITY_KEY: $value = $this->settingsService->getSensitivity(); break; + case SettingsService::DEVIATION_KEY: + $value = $this->settingsService->getDeviation(); + break; case SettingsService::MINIMUM_CONFIDENCE_KEY: $value = $this->settingsService->getMinimumConfidence(); break; diff --git a/lib/Db/FaceMapper.php b/lib/Db/FaceMapper.php index a63d3a80..c7fa9223 100644 --- a/lib/Db/FaceMapper.php +++ b/lib/Db/FaceMapper.php @@ -31,8 +31,6 @@ use OCP\AppFramework\Db\DoesNotExistException; use OCP\DB\QueryBuilder\IQueryBuilder; -use OCA\FaceRecognition\Helper\Euclidean; - class FaceMapper extends QBMapper { public function __construct(IDBConnection $db) { parent::__construct($db, 'facerecog_faces', '\OCA\FaceRecognition\Db\Face'); @@ -134,6 +132,9 @@ public function findFacesFromPerson(string $userId, int $personId, int $model, $ } public function findRepresentativeFromPerson(string $userId, int $personId, float $sensitivity, int $model) { + if (!version_compare(phpversion('pdlib'), '1.0.2', '>=')) { + return null; + } $qb = $this->db->getQueryBuilder(); $qb->select('f.id', 'f.image', 'f.person', 'f.descriptor') ->from($this->getTableName(), 'f') @@ -144,12 +145,11 @@ public function findRepresentativeFromPerson(string $userId, int $personId, floa $faces = $this->findEntities($qb); $facesCount = array(); - $euclidean = new Euclidean(); for ($i = 0, $face_count1 = count($faces); $i < $face_count1; $i++) { $face1 = $faces[$i]; for ($j = $i, $face_count2 = count($faces); $j < $face_count2; $j++) { $face2 = $faces[$j]; - $distance = $euclidean->distance($face1->descriptor, $face2->descriptor); + $distance = dlib_vector_length($face1->descriptor, $face2->descriptor); if ($distance < $sensitivity) { if (!array_key_exists($i, $facesCount)) { $facesCount[$i] = 1; diff --git a/lib/Service/SettingsService.php b/lib/Service/SettingsService.php index 6878fd26..8c9a9938 100644 --- a/lib/Service/SettingsService.php +++ b/lib/Service/SettingsService.php @@ -57,6 +57,12 @@ class SettingsService { const DEFAULT_SENSITIVITY = '0.4'; const MAXIMUM_SENSITIVITY = '0.6'; + /** Deviation used to suggestions */ + const DEVIATION_KEY = 'deviation'; + const MINIMUM_DEVIATION = '0.0'; + const DEFAULT_DEVIATION = '0.0'; + const MAXIMUM_DEVIATION = '0.2'; + /** Minimum confidence used to try to clustring faces */ const MINIMUM_CONFIDENCE_KEY = 'min_confidence'; const MINIMUM_MINIMUM_CONFIDENCE = '0.0'; @@ -220,6 +226,14 @@ public function setSensitivity($sensitivity) { $this->config->setAppValue(Application::APP_NAME, self::SENSITIVITY_KEY, $sensitivity); } + public function getDeviation(): float { + return floatval($this->config->getAppValue(Application::APP_NAME, self::DEVIATION_KEY, self::DEFAULT_DEVIATION)); + } + + public function setDeviation($deviation) { + $this->config->setAppValue(Application::APP_NAME, self::DEVIATION_KEY, $deviation); + } + public function getMinimumConfidence(): float { return floatval($this->config->getAppValue(Application::APP_NAME, self::MINIMUM_CONFIDENCE_KEY, self::DEFAULT_MINIMUM_CONFIDENCE)); } diff --git a/templates/settings/admin.php b/templates/settings/admin.php index e8a9e680..beffdb24 100644 --- a/templates/settings/admin.php +++ b/templates/settings/admin.php @@ -40,6 +40,21 @@


+

+ t('Deviation for suggestions'));?> +

+

t('The deviation is numerically added to the sensitivity to compare the clusters obtained by that sensitivity and suggest renaming similar clusters.'));?> + +

+

+ + + + ... + + +

+

t('Minimum confidence'));?>

From e73d272de461e344e2a330cb67ba20c3754b6eb8 Mon Sep 17 00:00:00 2001 From: Matias De lellis Date: Sun, 19 Apr 2020 13:31:20 -0300 Subject: [PATCH 03/16] ot show any message to the user if the suggestion is disabled --- js/personal.js | 8 ++++---- js/similar.js | 24 ++++++++++++++++++------ lib/Controller/PersonController.php | 17 +++++++++++++---- 3 files changed, 35 insertions(+), 14 deletions(-) diff --git a/js/personal.js b/js/personal.js index 6cd589fd..b9886bba 100644 --- a/js/personal.js +++ b/js/personal.js @@ -167,15 +167,15 @@ View.prototype = { self._persons.renameCluster (personId, name).done(function() { self._persons.unsetActive(); self.renderContent(); - if (true) { - self._similar.loadSimilar(personId, name).done(function() { + self._similar.loadSimilar(personId, name).done(function() { + if (self._similar.isEnabled()) { if (self._similar.hasSuggestion()) { self.suggestPerson(self._similar.getSuggestion(), name); } else { OC.Notification.showTemporary(t('facerecognition', 'There are no more suggestions from similar persons')); } - }); - } + } + }); }).fail(function () { OC.Notification.showTemporary(t('facerecognition', 'There was an error renaming this person')); }); diff --git a/js/similar.js b/js/similar.js index f376de0e..a95c88e5 100644 --- a/js/similar.js +++ b/js/similar.js @@ -1,22 +1,29 @@ var Similar = function (baseUrl) { this._baseUrl = baseUrl; + this._enabled = false; this._similarClusters = []; this._similarRejected = []; this._similarName = undefined; }; Similar.prototype = { + isEnabled: function () { + return this._enabled; + }, loadSimilar: function (clusterId, clusterName) { if (this._similarName != clusterName) { - this._similarClusters = []; - this._similarRejected = []; - this._similarName = undefined; + this.resetSuggestions(); } var self = this; var deferred = $.Deferred(); - $.get(this._baseUrl+'/clusters/similar/'+clusterId).done(function (similarClusters) { - self.concatNewClusters(similarClusters); - self._similarName = clusterName; + $.get(this._baseUrl+'/clusters/similar/'+clusterId).done(function (response) { + self._enabled = response.enabled; + if (!self._enabled) { + self.resetSuggestions(); + } else { + self.concatNewClusters(response.suggestions); + self._similarName = clusterName; + } deferred.resolve(); }).fail(function () { deferred.reject(); @@ -41,4 +48,9 @@ Similar.prototype = { } }); }, + resetSuggestions: function () { + this._similarClusters = []; + this._similarRejected = []; + this._similarName = undefined; + }, }; \ No newline at end of file diff --git a/lib/Controller/PersonController.php b/lib/Controller/PersonController.php index 952a9565..fc9dc983 100644 --- a/lib/Controller/PersonController.php +++ b/lib/Controller/PersonController.php @@ -176,8 +176,15 @@ public function find(int $id) { */ public function findSimilar(int $id) { $deviation = $this->settingsService->getDeviation(); - if (!version_compare(phpversion('pdlib'), '1.0.2', '>=') || ($deviation === 0.0)) - return new DataResponse(array()); + + $enabled = (version_compare(phpversion('pdlib'), '1.0.2', '>=') && ($deviation > 0.0)); + + $resp = array(); + $resp['enabled'] = $enabled; + $resp['suggestions'] = array(); + + if (!$enabled) + return new DataResponse($resp); $sensitivity = $this->settingsService->getSensitivity(); $modelId = $this->settingsService->getCurrentFaceModel(); @@ -185,7 +192,7 @@ public function findSimilar(int $id) { $mainPerson = $this->personMapper->find($this->userId, $id); $mainFace = $this->faceMapper->findRepresentativeFromPerson($this->userId, $id, $sensitivity, $modelId); - $resp = array(); + $suggestions = array(); $persons = $this->personMapper->findAll($this->userId, $modelId); foreach ($persons as $cmpPerson) { if ($mainPerson->getName() === $cmpPerson->getName()) @@ -198,10 +205,12 @@ public function findSimilar(int $id) { $similar['id'] = $cmpPerson->getId(); $similar['name'] = $cmpPerson->getName(); $similar['thumb-url'] = $this->getThumbUrl($cmpFace->getId()); - $resp[] = $similar; + $suggestions[] = $similar; } } + $resp['suggestions'] = $suggestions; + return new DataResponse($resp); } From 200d3c8423744b260e765a68bb57280176f45be1 Mon Sep 17 00:00:00 2001 From: Matias De lellis Date: Tue, 21 Apr 2020 10:26:39 -0300 Subject: [PATCH 04/16] Add Relation table, and fill after create clusters --- appinfo/info.xml | 2 +- .../Tasks/CreateClustersTask.php | 49 +++++++++++- lib/Db/Relation.php | 68 +++++++++++++++++ lib/Db/RelationMapper.php | 75 +++++++++++++++++++ .../Version000516Date20200420003814.php | 66 ++++++++++++++++ 5 files changed, 257 insertions(+), 3 deletions(-) create mode 100644 lib/Db/Relation.php create mode 100644 lib/Db/RelationMapper.php create mode 100644 lib/Migration/Version000516Date20200420003814.php diff --git a/appinfo/info.xml b/appinfo/info.xml index 8f539794..085e414c 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -16,7 +16,7 @@ - **🚀 Build your own thing:** FaceRecognition app is just a basic building block. Through FaceRecognition API, you can build your advanced scenarios - automatically add tags to images, connect contacts and persons, share images from specific person… We want to hear your ideas! ]]> - 0.5.15 + 0.5.16 agpl Matias De lellis Branko Kokanovic diff --git a/lib/BackgroundJob/Tasks/CreateClustersTask.php b/lib/BackgroundJob/Tasks/CreateClustersTask.php index 9f12faac..884df28e 100644 --- a/lib/BackgroundJob/Tasks/CreateClustersTask.php +++ b/lib/BackgroundJob/Tasks/CreateClustersTask.php @@ -32,6 +32,9 @@ use OCA\FaceRecognition\Db\ImageMapper; use OCA\FaceRecognition\Db\PersonMapper; +use OCA\FaceRecognition\Db\Relation; +use OCA\FaceRecognition\Db\RelationMapper; + use OCA\FaceRecognition\Helper\Euclidean; use OCA\FaceRecognition\Service\SettingsService; @@ -48,6 +51,9 @@ class CreateClustersTask extends FaceRecognitionBackgroundTask { /** @var FaceMapper Face mapper*/ private $faceMapper; + /** @var RelationMapper Relation mapper*/ + private $relationMapper; + /** @var SettingsService Settings service*/ private $settingsService; @@ -60,6 +66,7 @@ class CreateClustersTask extends FaceRecognitionBackgroundTask { public function __construct(PersonMapper $personMapper, ImageMapper $imageMapper, FaceMapper $faceMapper, + RelationMapper $relationMapper, SettingsService $settingsService) { parent::__construct(); @@ -67,6 +74,7 @@ public function __construct(PersonMapper $personMapper, $this->personMapper = $personMapper; $this->imageMapper = $imageMapper; $this->faceMapper = $faceMapper; + $this->relationMapper = $relationMapper; $this->settingsService = $settingsService; } @@ -207,8 +215,10 @@ private function createClusterIfNeeded(string $userId) { $this->logInfo('Deleted ' . $orphansDeleted . ' persons without faces'); } - // Prevents not create/recreate the clusters unnecessarily. + // Fill relation table with new clusters. + $this->fillFaceRelationsFromPersons($userId); + // Prevents not create/recreate the clusters unnecessarily. $this->settingsService->setNeedRecreateClusters(false, $userId); $this->settingsService->setForceCreateClusters(false, $userId); } @@ -228,7 +238,6 @@ private function getCurrentClusters(array $faces): array { private function getNewClusters(array $faces): array { // Create edges for chinese whispers - $euclidean = new Euclidean(); $sensitivity = $this->settingsService->getSensitivity(); $min_confidence = $this->settingsService->getMinimumConfidence(); $edges = array(); @@ -250,6 +259,7 @@ private function getNewClusters(array $faces): array { } } } else { + $euclidean = new Euclidean(); for ($i = 0, $face_count1 = count($faces); $i < $face_count1; $i++) { $face1 = $faces[$i]; if ($face1->confidence < $min_confidence) { @@ -343,4 +353,39 @@ public function mergeClusters(array $oldCluster, array $newCluster): array { } return $result; } + + private function fillFaceRelationsFromPersons(string $userId) { + $deviation = $this->settingsService->getDeviation(); + if (!version_compare(phpversion('pdlib'), '1.0.2', '>=') || ($deviation === 0.0)) + return; + + $sensitivity = $this->settingsService->getSensitivity(); + $modelId = $this->settingsService->getCurrentFaceModel(); + + // Get the representative faces of each person + $mainFaces = array(); + $persons = $this->personMapper->findAll($userId, $modelId); + foreach ($persons as $person) { + $mainFaces[] = $this->faceMapper->findRepresentativeFromPerson($userId, $person->getId(), $sensitivity, $modelId); + } + + // Get similar faces taking into account the deviation and insert new relations + for ($i = 0, $face_count1 = count($mainFaces); $i < $face_count1; $i++) { + $face1 = $mainFaces[$i]; + for ($j = $i+1, $face_count2 = count($mainFaces); $j < $face_count2; $j++) { + $face2 = $mainFaces[$j]; + $distance = dlib_vector_length($face1->descriptor, $face2->descriptor); + if ($distance < ($sensitivity + $deviation)) { + $relation = new Relation(); + $relation->setFace1($face1->getId()); + $relation->setFace2($face2->getId()); + $relation->setState(RELATION::PROPOSED); + if (!$this->relationMapper->exists($relation)) { + $this->relationMapper->insert($relation); + } + } + } + } + } + } diff --git a/lib/Db/Relation.php b/lib/Db/Relation.php new file mode 100644 index 00000000..b2817336 --- /dev/null +++ b/lib/Db/Relation.php @@ -0,0 +1,68 @@ + + * + * @author Matias De lellis + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ +namespace OCA\FaceRecognition\Db; + +use OCP\AppFramework\Db\Entity; + +/** + * Relation represents one relation beetwen two faces + * + * @method int getFace1() + * @method int getFace2() + * @method int getState() + * @method void setFace1(int $face1) + * @method void setFace2(int $face2) + * @method void setState(int $state) + */ +class Relation extends Entity { + + /** + * Possible values of the state of a face relation + */ + public const PROPOSED = 0; + public const ACCEPTED = 1; + public const REJECTED = 2; + + /** + * Face id of a face of a person related with $face2 + * + * @var int + * */ + protected $face1; + + /** + * Face id of a face of a person related with $face1 + * + * @var int + * */ + protected $face2; + + /** + * State of two face relation. These are proposed, and can be accepted + * as as the same person, or rejected. + * + * @var int + * */ + protected $state; + +} \ No newline at end of file diff --git a/lib/Db/RelationMapper.php b/lib/Db/RelationMapper.php new file mode 100644 index 00000000..00d7852e --- /dev/null +++ b/lib/Db/RelationMapper.php @@ -0,0 +1,75 @@ + + * + * @author Matias De lellis + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCA\FaceRecognition\Db; + +use OC\DB\QueryBuilder\Literal; + +use OCP\IDBConnection; +use OCP\AppFramework\Db\QBMapper; +use OCP\AppFramework\Db\DoesNotExistException; +use OCP\DB\QueryBuilder\IQueryBuilder; + +class RelationMapper extends QBMapper { + + public function __construct(IDBConnection $db) { + parent::__construct($db, 'facerecog_relations', '\OCA\FaceRecognition\Db\Relation'); + } + + public function exists(Relation $relation): bool { + $qb = $this->db->getQueryBuilder(); + $query = $qb + ->select(['id']) + ->from($this->getTableName()) + ->where($qb->expr()->eq('face1', $qb->createParameter('face1'))) + ->andWhere($qb->expr()->eq('face2', $qb->createParameter('face2'))) + ->setParameter('face1', $relation->getFace1()) + ->setParameter('face2', $relation->getFace2()); + + $resultStatement = $query->execute(); + $row = $resultStatement->fetch(); + $resultStatement->closeCursor(); + + return ($row !== false); + } + + /*public function findFromPerson(string $userId, int $personId, int $model, int $state = null): array { + $qb = $this->db->getQueryBuilder(); + $qb->select('r.id', 'r.face1', 'r.face1', 'r.state') + ->from($this->getTableName(), 'r') + ->innerJoin('f', 'facerecog_images' ,'i', $qb->expr()->eq('f.image', 'i.id')) + ->innerJoin('f', 'facerecog_images' ,'i', $qb->expr()->eq('f.image', 'i.id')) + ->where($qb->expr()->eq('user', $qb->createNamedParameter($userId))) + ->andWhere($qb->expr()->eq('person', $qb->createNamedParameter($personId))) + ->andWhere($qb->expr()->eq('model', $qb->createNamedParameter($model))); + + if (!is_null($state)) { + $qb->andWhere($qb->expr()->eq('state', $qb->createNamedParameter($state))); + } + + $relations = $this->findEntities($qb); + + return $relations; + }*/ + +} \ No newline at end of file diff --git a/lib/Migration/Version000516Date20200420003814.php b/lib/Migration/Version000516Date20200420003814.php new file mode 100644 index 00000000..1b2de535 --- /dev/null +++ b/lib/Migration/Version000516Date20200420003814.php @@ -0,0 +1,66 @@ +hasTable('facerecog_relations')) { + $table = $schema->createTable('facerecog_relations'); + $table->addColumn('id', 'integer', [ + 'autoincrement' => true, + 'notnull' => true, + 'unsigned' => true, + ]); + $table->addColumn('face1', 'integer', [ + 'notnull' => true, + 'length' => 4, + ]); + $table->addColumn('face2', 'integer', [ + 'notnull' => true, + 'length' => 4, + ]); + $table->addColumn('state', 'integer', [ + 'notnull' => true, + 'length' => 4, + ]); + + $table->setPrimaryKey(['id']); + $table->addIndex(['face1'], 'relation_faces_1_idx'); + $table->addIndex(['face2'], 'relation_faces_2_idx'); + } + return $schema; + } + + /** + * @param IOutput $output + * @param Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper` + * @param array $options + */ + public function postSchemaChange(IOutput $output, Closure $schemaClosure, array $options) { + } +} From 1dc82a3c8e2603f75fe4ed2d1865832b70da497a Mon Sep 17 00:00:00 2001 From: Matias De lellis Date: Tue, 21 Apr 2020 13:14:19 -0300 Subject: [PATCH 05/16] Migrate suggestions to proposed as relation --- lib/Controller/PersonController.php | 36 +++++++++++++++++------------ lib/Db/PersonMapper.php | 18 +++++++++++++++ lib/Db/RelationMapper.php | 26 ++++++++------------- 3 files changed, 49 insertions(+), 31 deletions(-) diff --git a/lib/Controller/PersonController.php b/lib/Controller/PersonController.php index fc9dc983..d7991406 100644 --- a/lib/Controller/PersonController.php +++ b/lib/Controller/PersonController.php @@ -44,6 +44,9 @@ use OCA\FaceRecognition\Db\Person; use OCA\FaceRecognition\Db\PersonMapper; +use OCA\FaceRecognition\Db\Relation; +use OCA\FaceRecognition\Db\RelationMapper; + use OCA\FaceRecognition\Service\SettingsService; class PersonController extends Controller { @@ -66,6 +69,9 @@ class PersonController extends Controller { /** @var PersonMapper */ private $personMapper; + /** @var RelationMapper */ + private $relationMapper; + /** @var SettingsService */ private $settingsService; @@ -80,6 +86,7 @@ public function __construct($AppName, FaceMapper $faceMapper, ImageMapper $imageMapper, PersonMapper $personmapper, + RelationMapper $relationMapper, SettingsService $settingsService, $UserId) { @@ -91,6 +98,7 @@ public function __construct($AppName, $this->faceMapper = $faceMapper; $this->imageMapper = $imageMapper; $this->personMapper = $personmapper; + $this->relationMapper = $relationMapper; $this->settingsService = $settingsService; $this->userId = $UserId; } @@ -186,25 +194,23 @@ public function findSimilar(int $id) { if (!$enabled) return new DataResponse($resp); - $sensitivity = $this->settingsService->getSensitivity(); - $modelId = $this->settingsService->getCurrentFaceModel(); - $mainPerson = $this->personMapper->find($this->userId, $id); - $mainFace = $this->faceMapper->findRepresentativeFromPerson($this->userId, $id, $sensitivity, $modelId); $suggestions = array(); - $persons = $this->personMapper->findAll($this->userId, $modelId); - foreach ($persons as $cmpPerson) { - if ($mainPerson->getName() === $cmpPerson->getName()) - continue; - - $cmpFace = $this->faceMapper->findRepresentativeFromPerson($this->userId, $cmpPerson->getId(), $sensitivity, $modelId); - $distance = dlib_vector_length($mainFace->descriptor, $cmpFace->descriptor); - if ($distance < ($sensitivity + $deviation)) { + $relations = $this->relationMapper->findFromPerson($this->userId, $id, RELATION::PROPOSED); + foreach ($relations as $relation) { + $person1 = $this->personMapper->findFromFace($this->userId, $relation->getFace1()); + if (($person1->getId() !== $id) && ($mainPerson->getName() !== $person1->getName())) { + $similar = array(); + $similar['id'] = $person1->getId(); + $similar['name'] = $person1->getName(); + $suggestions[] = $similar; + } + $person2 = $this->personMapper->findFromFace($this->userId, $relation->getFace2()); + if (($person2->getId() !== $id) && ($mainPerson->getName() !== $person2->getName())) { $similar = array(); - $similar['id'] = $cmpPerson->getId(); - $similar['name'] = $cmpPerson->getName(); - $similar['thumb-url'] = $this->getThumbUrl($cmpFace->getId()); + $similar['id'] = $person2->getId(); + $similar['name'] = $person2->getName(); $suggestions[] = $similar; } } diff --git a/lib/Db/PersonMapper.php b/lib/Db/PersonMapper.php index 3cfb4ee5..3850bcf6 100644 --- a/lib/Db/PersonMapper.php +++ b/lib/Db/PersonMapper.php @@ -102,6 +102,24 @@ public function findAll(string $userId, int $modelId): array { return $this->findEntities($qb); } + /** + * Find a person that contains a face. + * + * @param string $userId ID of the user + * @param int $faceId ID of the face that belongs to the wanted person + * @return Person + */ + public function findFromFace(string $userId, int $faceId) { + $qb = $this->db->getQueryBuilder(); + $qb->select('p.id', 'p.name', 'p.is_valid') + ->from($this->getTableName(), 'p') + ->innerJoin('p', 'facerecog_faces' ,'f', $qb->expr()->eq('p.id', 'f.person')) + ->where($qb->expr()->eq('p.user', $qb->createNamedParameter($userId))) + ->andWhere($qb->expr()->eq('f.id', $qb->createNamedParameter($faceId))); + + return $this->findEntity($qb); + } + /** * Returns count of persons (clusters) found for a given user. * diff --git a/lib/Db/RelationMapper.php b/lib/Db/RelationMapper.php index 00d7852e..da7c6997 100644 --- a/lib/Db/RelationMapper.php +++ b/lib/Db/RelationMapper.php @@ -53,23 +53,17 @@ public function exists(Relation $relation): bool { return ($row !== false); } - /*public function findFromPerson(string $userId, int $personId, int $model, int $state = null): array { + public function findFromPerson(string $userId, int $personId, int $state): array { $qb = $this->db->getQueryBuilder(); - $qb->select('r.id', 'r.face1', 'r.face1', 'r.state') - ->from($this->getTableName(), 'r') - ->innerJoin('f', 'facerecog_images' ,'i', $qb->expr()->eq('f.image', 'i.id')) - ->innerJoin('f', 'facerecog_images' ,'i', $qb->expr()->eq('f.image', 'i.id')) - ->where($qb->expr()->eq('user', $qb->createNamedParameter($userId))) - ->andWhere($qb->expr()->eq('person', $qb->createNamedParameter($personId))) - ->andWhere($qb->expr()->eq('model', $qb->createNamedParameter($model))); + $qb->select('r.id', 'r.face1', 'r.face2', 'r.state') + ->from($this->getTableName(), 'r') + ->innerJoin('r', 'facerecog_faces' ,'f', $qb->expr()->orX($qb->expr()->eq('r.face1', 'f.id'), $qb->expr()->eq('r.face2', 'f.id'))) + ->innerJoin('f', 'facerecog_persons' ,'p', $qb->expr()->eq('f.person', 'p.id')) + ->where($qb->expr()->eq('p.user', $qb->createNamedParameter($userId))) + ->andWhere($qb->expr()->eq('p.id', $qb->createNamedParameter($personId))) + ->andWhere($qb->expr()->eq('r.state', $qb->createNamedParameter($state))); - if (!is_null($state)) { - $qb->andWhere($qb->expr()->eq('state', $qb->createNamedParameter($state))); - } - - $relations = $this->findEntities($qb); - - return $relations; - }*/ + return $this->findEntities($qb); + } } \ No newline at end of file From 3431b13d22f6db189d70ca3e1c1a87897710c79c Mon Sep 17 00:00:00 2001 From: Matias De lellis Date: Tue, 21 Apr 2020 23:25:16 -0300 Subject: [PATCH 06/16] Add Relation controller and use it --- appinfo/routes.php | 18 ++-- js/personal.js | 49 +++++---- js/similar.js | 70 +++++++++---- lib/Controller/PersonController.php | 49 --------- lib/Controller/RelationController.php | 141 ++++++++++++++++++++++++++ lib/Db/RelationMapper.php | 22 ++++ 6 files changed, 253 insertions(+), 96 deletions(-) create mode 100644 lib/Controller/RelationController.php diff --git a/appinfo/routes.php b/appinfo/routes.php index 740c97d6..4d402f9d 100644 --- a/appinfo/routes.php +++ b/appinfo/routes.php @@ -7,12 +7,6 @@ 'url' => '/clusters', 'verb' => 'GET' ], - // Get a similar clusters by Id. - [ - 'name' => 'person#findSimilar', - 'url' => '/clusters/similar/{id}', - 'verb' => 'GET' - ], // Get all clusters filtered by Name. [ 'name' => 'person#findByName', @@ -43,6 +37,18 @@ 'url' => '/file', 'verb' => 'GET' ], + // Get proposed relations on an person. + [ + 'name' => 'relation#findByPerson', + 'url' => '/relation/{personId}', + 'verb' => 'GET' + ], + // Change an relation of an person. + [ + 'name' => 'relation#updateByPersons', + 'url' => '/relation/{personId}', + 'verb' => 'PUT' + ], // Get folder preferences [ 'name' => 'file#getFolderOptions', diff --git a/js/personal.js b/js/personal.js index b9886bba..f18e630b 100644 --- a/js/personal.js +++ b/js/personal.js @@ -99,7 +99,7 @@ Persons.prototype = { getAll: function () { return this._clusters; }, - renameCluster: function (clusterId, personName) { + updateCluster: function (clusterId, personName) { var self = this; var deferred = $.Deferred(); var opt = { name: personName }; @@ -108,16 +108,19 @@ Persons.prototype = { contentType: 'application/json', data: JSON.stringify(opt) }).done(function (data) { - self._clusters.forEach(function (cluster) { - if (cluster.id === clusterId) { - cluster.name = personName; - } - }); + self.renameCluster(clusterId, personName); deferred.resolve(); }).fail(function () { deferred.reject(); }); return deferred.promise(); + }, + renameCluster: function (clusterId, personName) { + this._clusters.forEach(function (cluster) { + if (cluster.id === clusterId) { + cluster.name = personName; + } + }); } }; @@ -164,13 +167,13 @@ View.prototype = { faceUrl, function(result, name) { if (result === true && name) { - self._persons.renameCluster (personId, name).done(function() { + self._persons.updateCluster(personId, name).done(function() { self._persons.unsetActive(); self.renderContent(); - self._similar.loadSimilar(personId, name).done(function() { + self._similar.findProposal(personId, name).done(function() { if (self._similar.isEnabled()) { - if (self._similar.hasSuggestion()) { - self.suggestPerson(self._similar.getSuggestion(), name); + if (self._similar.hasProposal()) { + self.suggestPerson(self._similar.getProposal(), name); } else { OC.Notification.showTemporary(t('facerecognition', 'There are no more suggestions from similar persons')); } @@ -183,19 +186,20 @@ View.prototype = { } ); }, - suggestPerson: function (suggestion, personName) { + suggestPerson: function (proposal, personName) { var self = this; FrDialogs.suggestPersonName( personName, - this._persons.getById(suggestion.id).faces, + this._persons.getById(proposal.id).faces, function(accepted, close) { if (accepted === true) { - self._persons.renameCluster (suggestion.id, personName).done(function() { + self._similar.acceptProposed(proposal).done(function() { + self._persons.renameCluster(proposal.id, personName); self._persons.unsetActive(); self.renderContent(); - self._similar.loadSimilar(suggestion.id, personName).done(function() { - if (self._similar.hasSuggestion()) { - self.suggestPerson(self._similar.getSuggestion(), personName); + self._similar.findProposal(proposal.id, personName).done(function() { + if (self._similar.hasProposal()) { + self.suggestPerson(self._similar.getProposal(), personName); } else { OC.Notification.showTemporary(t('facerecognition', 'There are no more suggestions from similar persons')); } @@ -204,12 +208,13 @@ View.prototype = { OC.Notification.showTemporary(t('facerecognition', 'There was an error renaming this person')); }); } else if (close === false) { - self._similar.rejectSuggestion(suggestion); - if (self._similar.hasSuggestion()) { - self.suggestPerson(self._similar.getSuggestion(), personName); - } else { - OC.Notification.showTemporary(t('facerecognition', 'There are no more suggestions from similar persons')); - } + self._similar.rejectProposed(proposal).done(function() { + if (self._similar.hasProposal()) { + self.suggestPerson(self._similar.getProposal(), personName); + } else { + OC.Notification.showTemporary(t('facerecognition', 'There are no more suggestions from similar persons')); + } + }); } } ); diff --git a/js/similar.js b/js/similar.js index a95c88e5..fbe0faad 100644 --- a/js/similar.js +++ b/js/similar.js @@ -1,7 +1,16 @@ +'use strict'; + +const Relation = { + PROPOSED: 0, + ACCEPTED: 1, + REJECTED: 2 +} + var Similar = function (baseUrl) { this._baseUrl = baseUrl; + this._enabled = false; - this._similarClusters = []; + this._similarProposed = []; this._similarRejected = []; this._similarName = undefined; }; @@ -10,18 +19,18 @@ Similar.prototype = { isEnabled: function () { return this._enabled; }, - loadSimilar: function (clusterId, clusterName) { - if (this._similarName != clusterName) { - this.resetSuggestions(); + findProposal: function (clusterId, clusterName) { + if (this._similarName !== clusterName) { + this.resetProposals(); } var self = this; var deferred = $.Deferred(); - $.get(this._baseUrl+'/clusters/similar/'+clusterId).done(function (response) { + $.get(this._baseUrl+'/relation/'+clusterId).done(function (response) { self._enabled = response.enabled; if (!self._enabled) { self.resetSuggestions(); } else { - self.concatNewClusters(response.suggestions); + self.concatNewProposals(response.proposed); self._similarName = clusterName; } deferred.resolve(); @@ -30,26 +39,49 @@ Similar.prototype = { }); return deferred.promise(); }, - hasSuggestion: function () { - return (this._similarClusters.length > 0); + acceptProposed: function (proposed) { + return this.updateRelation(proposed.origId, proposed.id, Relation.ACCEPTED); }, - getSuggestion: function () { - return this._similarClusters.shift(); + rejectProposed: function (proposed) { + this._similarRejected.push(proposed); + return this.updateRelation(proposed.origId, proposed.id, Relation.REJECTED); }, - rejectSuggestion: function (suggestion) { - return this._similarRejected.push(suggestion); + hasProposal: function () { + return (this._similarProposed.length > 0); + }, + getProposal: function () { + return this._similarProposed.shift(); + }, + updateRelation: function (clusterId, toClusterId, newState) { + var self = this; + var deferred = $.Deferred(); + var data = { + toPersonId: toClusterId, + state: newState + }; + $.ajax({ + url: this._baseUrl + '/relation/' + clusterId, + method: 'PUT', + contentType: 'application/json', + data: JSON.stringify(data) + }).done(function (data) { + deferred.resolve(); + }).fail(function () { + deferred.reject(); + }); + return deferred.promise(); }, - concatNewClusters: function (newClusters) { + concatNewProposals: function (proposals) { var self = this; - newClusters.forEach(function (newCluster) { - if ((self._similarClusters.find(function (oldCluster) { return newCluster.id === oldCluster.id;}) === undefined) && - (self._similarRejected.find(function (rejCluster) { return newCluster.id === rejCluster.id;}) === undefined)) { - self._similarClusters.push(newCluster); + proposals.forEach(function (proposed) { + if ((self._similarProposed.find(function (oldProposed) { return proposed.id === oldProposed.id;}) === undefined) && + (self._similarRejected.find(function (rejProposed) { return proposed.id === rejProposed.id;}) === undefined)) { + self._similarProposed.push(proposed); } }); }, - resetSuggestions: function () { - this._similarClusters = []; + resetProposals: function () { + this._similarProposed = []; this._similarRejected = []; this._similarName = undefined; }, diff --git a/lib/Controller/PersonController.php b/lib/Controller/PersonController.php index d7991406..2fab81c6 100644 --- a/lib/Controller/PersonController.php +++ b/lib/Controller/PersonController.php @@ -44,9 +44,6 @@ use OCA\FaceRecognition\Db\Person; use OCA\FaceRecognition\Db\PersonMapper; -use OCA\FaceRecognition\Db\Relation; -use OCA\FaceRecognition\Db\RelationMapper; - use OCA\FaceRecognition\Service\SettingsService; class PersonController extends Controller { @@ -69,9 +66,6 @@ class PersonController extends Controller { /** @var PersonMapper */ private $personMapper; - /** @var RelationMapper */ - private $relationMapper; - /** @var SettingsService */ private $settingsService; @@ -86,7 +80,6 @@ public function __construct($AppName, FaceMapper $faceMapper, ImageMapper $imageMapper, PersonMapper $personmapper, - RelationMapper $relationMapper, SettingsService $settingsService, $UserId) { @@ -98,7 +91,6 @@ public function __construct($AppName, $this->faceMapper = $faceMapper; $this->imageMapper = $imageMapper; $this->personMapper = $personmapper; - $this->relationMapper = $relationMapper; $this->settingsService = $settingsService; $this->userId = $UserId; } @@ -179,47 +171,6 @@ public function find(int $id) { return new DataResponse($resp); } - /** - * @NoAdminRequired - */ - public function findSimilar(int $id) { - $deviation = $this->settingsService->getDeviation(); - - $enabled = (version_compare(phpversion('pdlib'), '1.0.2', '>=') && ($deviation > 0.0)); - - $resp = array(); - $resp['enabled'] = $enabled; - $resp['suggestions'] = array(); - - if (!$enabled) - return new DataResponse($resp); - - $mainPerson = $this->personMapper->find($this->userId, $id); - - $suggestions = array(); - $relations = $this->relationMapper->findFromPerson($this->userId, $id, RELATION::PROPOSED); - foreach ($relations as $relation) { - $person1 = $this->personMapper->findFromFace($this->userId, $relation->getFace1()); - if (($person1->getId() !== $id) && ($mainPerson->getName() !== $person1->getName())) { - $similar = array(); - $similar['id'] = $person1->getId(); - $similar['name'] = $person1->getName(); - $suggestions[] = $similar; - } - $person2 = $this->personMapper->findFromFace($this->userId, $relation->getFace2()); - if (($person2->getId() !== $id) && ($mainPerson->getName() !== $person2->getName())) { - $similar = array(); - $similar['id'] = $person2->getId(); - $similar['name'] = $person2->getName(); - $suggestions[] = $similar; - } - } - - $resp['suggestions'] = $suggestions; - - return new DataResponse($resp); - } - /** * @NoAdminRequired */ diff --git a/lib/Controller/RelationController.php b/lib/Controller/RelationController.php new file mode 100644 index 00000000..e407b25d --- /dev/null +++ b/lib/Controller/RelationController.php @@ -0,0 +1,141 @@ + + * + * @author Matias De lellis + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCA\FaceRecognition\Controller; + +use OCP\IRequest; + +use OCP\AppFramework\Http; +use OCP\AppFramework\Http\DataResponse; +use OCP\AppFramework\Http\JSONResponse; +use OCP\AppFramework\Http\DataDisplayResponse; +use OCP\AppFramework\Controller; + +use OCA\FaceRecognition\Db\Person; +use OCA\FaceRecognition\Db\PersonMapper; + +use OCA\FaceRecognition\Db\Relation; +use OCA\FaceRecognition\Db\RelationMapper; + +use OCA\FaceRecognition\Service\SettingsService; + +class RelationController extends Controller { + + /** @var PersonMapper */ + private $personMapper; + + /** @var RelationMapper */ + private $relationMapper; + + /** @var SettingsService */ + private $settingsService; + + /** @var string */ + private $userId; + + public function __construct($AppName, + IRequest $request, + PersonMapper $personMapper, + RelationMapper $relationMapper, + SettingsService $settingsService, + $UserId) + { + parent::__construct($AppName, $request); + + $this->personMapper = $personMapper; + $this->relationMapper = $relationMapper; + $this->settingsService = $settingsService; + $this->userId = $UserId; + } + + /** + * @NoAdminRequired + * @param int $personId + */ + public function findByPerson(int $personId) { + $deviation = $this->settingsService->getDeviation(); + + $enabled = (version_compare(phpversion('pdlib'), '1.0.2', '>=') && ($deviation > 0.0)); + + $resp = array(); + $resp['enabled'] = $enabled; + $resp['proposed'] = array(); + + if (!$enabled) + return new DataResponse($resp); + + $mainPerson = $this->personMapper->find($this->userId, $personId); + + $proposed = array(); + $relations = $this->relationMapper->findFromPerson($this->userId, $personId, RELATION::PROPOSED); + foreach ($relations as $relation) { + $person1 = $this->personMapper->findFromFace($this->userId, $relation->getFace1()); + if (($person1->getId() !== $personId) && ($mainPerson->getName() !== $person1->getName())) { + $proffer = array(); + $proffer['origId'] = $mainPerson->getId(); + $proffer['id'] = $person1->getId(); + $proffer['name'] = $person1->getName(); + $proposed[] = $proffer; + } + $person2 = $this->personMapper->findFromFace($this->userId, $relation->getFace2()); + if (($person2->getId() !== $personId) && ($mainPerson->getName() !== $person2->getName())) { + $proffer = array(); + $proffer['origId'] = $mainPerson->getId(); + $proffer['id'] = $person2->getId(); + $proffer['name'] = $person2->getName(); + $proposed[] = $proffer; + } + } + $resp['proposed'] = $proposed; + + return new DataResponse($resp); + } + + /** + * @NoAdminRequired + * @param int $personId + * @param int $toPersonId + * @param int $state + */ + public function updateByPersons(int $personId, int $toPersonId, int $state) { + $relations = $this->relationMapper->findFromPersons($personId, $toPersonId); + + foreach ($relations as $relation) { + $relation->setState($state); + $this->relationMapper->update($relation); + } + + if ($state === RELATION::ACCEPTED) { + $person = $this->personMapper->find($this->userId, $personId); + $name = $person->getName(); + + $toPerson = $this->personMapper->find($this->userId, $toPersonId); + $toPerson->setName($name); + $this->personMapper->update($toPerson); + } + + $relations = $this->relationMapper->findFromPersons($personId, $toPersonId); + return new DataResponse($relations); + } + +} diff --git a/lib/Db/RelationMapper.php b/lib/Db/RelationMapper.php index da7c6997..cd4674f1 100644 --- a/lib/Db/RelationMapper.php +++ b/lib/Db/RelationMapper.php @@ -66,4 +66,26 @@ public function findFromPerson(string $userId, int $personId, int $state): array return $this->findEntities($qb); } + public function findFromPersons(int $personId1, int $personId2) { + $sub1 = $this->db->getQueryBuilder(); + $sub1->select('f.id') + ->from('facerecog_faces', 'f') + ->where($sub1->expr()->eq('f.person', $sub1->createParameter('person1'))); + + $sub2 = $this->db->getQueryBuilder(); + $sub2->select('f.id') + ->from('facerecog_faces', 'f') + ->where($sub2->expr()->eq('f.person', $sub2->createParameter('person2'))); + + $qb = $this->db->getQueryBuilder(); + $qb->select('r.id', 'r.face1', 'r.face2', 'r.state') + ->from($this->getTableName(), 'r') + ->where('r.face1 IN (' . $sub1->getSQL() . ')') + ->orWhere('r.face2 IN (' . $sub2->getSQL() . ')') + ->setParameter('person1', $personId1) + ->setParameter('person2', $personId2); + + return $this->findEntities($qb); + } + } \ No newline at end of file From 218439cfa9aa8d84d34496eba9cf0e45079247d8 Mon Sep 17 00:00:00 2001 From: Matias De lellis Date: Wed, 22 Apr 2020 12:35:14 -0300 Subject: [PATCH 07/16] Add an true 'I am not sure' button, that not reject completely the proposal --- css/facerecognition.css | 5 +++++ js/fr-dialogs.js | 19 +++++++++++++------ js/personal.js | 38 ++++++++++++++++++++++---------------- js/similar.js | 15 +++++---------- 4 files changed, 45 insertions(+), 32 deletions(-) diff --git a/css/facerecognition.css b/css/facerecognition.css index 2f2eae94..b5550dc8 100644 --- a/css/facerecognition.css +++ b/css/facerecognition.css @@ -52,6 +52,11 @@ width: 50px; } + +.oc-dialog-buttonrow, .threebuttons { + justify-content: space-between; +} + /* * Admin page */ diff --git a/js/fr-dialogs.js b/js/fr-dialogs.js index 6b14d48a..bd1ed5c6 100644 --- a/js/fr-dialogs.js +++ b/js/fr-dialogs.js @@ -115,21 +115,28 @@ const FrDialogs = { } var buttonlist = [{ - text: t('facerecognition', 'I don\'t know'), + text: t('facerecognition', 'No'), click: function () { if (callback !== undefined) { $(dialogId).ocdialog('close'); } - callback(false, false); + callback(true, Relation.REJECTED); }, - defaultButton: false - },{ + }, { + text: t('facerecognition', 'I am not sure'), + click: function () { + if (callback !== undefined) { + $(dialogId).ocdialog('close'); + } + callback(true, Relation.PROPOSED); + } + }, { text: t('facerecognition', 'Yes'), click: function () { if (callback !== undefined) { $(dialogId).ocdialog('close'); } - callback(true, false); + callback(true, Relation.ACCEPTED); }, defaultButton: true }]; @@ -141,7 +148,7 @@ const FrDialogs = { close: function () { // callback is already fired if Yes/No is clicked directly if (callback !== undefined) { - callback(false, true); + callback(false, Relation.PROPOSED); } } }); diff --git a/js/personal.js b/js/personal.js index f18e630b..6dc77708 100644 --- a/js/personal.js +++ b/js/personal.js @@ -191,28 +191,34 @@ View.prototype = { FrDialogs.suggestPersonName( personName, this._persons.getById(proposal.id).faces, - function(accepted, close) { - if (accepted === true) { - self._similar.acceptProposed(proposal).done(function() { - self._persons.renameCluster(proposal.id, personName); - self._persons.unsetActive(); - self.renderContent(); - self._similar.findProposal(proposal.id, personName).done(function() { + function(valid, state) { + if (valid === true) { + // It is valid must be update the proposals. + self._similar.updateProposal(proposal, state).done(function() { + if (state === Relation.ACCEPTED) { + // Update view with new name. + self._persons.renameCluster(proposal.id, personName); + self._persons.unsetActive(); + self.renderContent(); + // Look for new suggestions based on accepted proposal + self._similar.findProposal(proposal.id, personName).done(function() { + if (self._similar.hasProposal()) { + self.suggestPerson(self._similar.getProposal(), personName); + } else { + OC.Notification.showTemporary(t('facerecognition', 'There are no more suggestions from similar persons')); + } + }); + } else { + // Suggest cached proposals if (self._similar.hasProposal()) { self.suggestPerson(self._similar.getProposal(), personName); } else { OC.Notification.showTemporary(t('facerecognition', 'There are no more suggestions from similar persons')); } - }); + } }).fail(function () { - OC.Notification.showTemporary(t('facerecognition', 'There was an error renaming this person')); - }); - } else if (close === false) { - self._similar.rejectProposed(proposal).done(function() { - if (self._similar.hasProposal()) { - self.suggestPerson(self._similar.getProposal(), personName); - } else { - OC.Notification.showTemporary(t('facerecognition', 'There are no more suggestions from similar persons')); + if (state === Relation.ACCEPTED) { + OC.Notification.showTemporary(t('facerecognition', 'There was an error renaming this person')); } }); } diff --git a/js/similar.js b/js/similar.js index fbe0faad..5a1362fc 100644 --- a/js/similar.js +++ b/js/similar.js @@ -39,32 +39,27 @@ Similar.prototype = { }); return deferred.promise(); }, - acceptProposed: function (proposed) { - return this.updateRelation(proposed.origId, proposed.id, Relation.ACCEPTED); - }, - rejectProposed: function (proposed) { - this._similarRejected.push(proposed); - return this.updateRelation(proposed.origId, proposed.id, Relation.REJECTED); - }, hasProposal: function () { return (this._similarProposed.length > 0); }, getProposal: function () { return this._similarProposed.shift(); }, - updateRelation: function (clusterId, toClusterId, newState) { + updateProposal: function (proposal, newState) { var self = this; var deferred = $.Deferred(); var data = { - toPersonId: toClusterId, + toPersonId: proposal.id, state: newState }; $.ajax({ - url: this._baseUrl + '/relation/' + clusterId, + url: this._baseUrl + '/relation/' + proposal.origId, method: 'PUT', contentType: 'application/json', data: JSON.stringify(data) }).done(function (data) { + if (newState !== Relation.ACCEPTED) + self._similarRejected.push(proposal); deferred.resolve(); }).fail(function () { deferred.reject(); From 18b209e28819414ee2a80252615fe434999f52e0 Mon Sep 17 00:00:00 2001 From: Matias De lellis Date: Wed, 22 Apr 2020 13:15:46 -0300 Subject: [PATCH 08/16] Fix query to find Relations between two persons. --- lib/Db/RelationMapper.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/Db/RelationMapper.php b/lib/Db/RelationMapper.php index cd4674f1..0079b3aa 100644 --- a/lib/Db/RelationMapper.php +++ b/lib/Db/RelationMapper.php @@ -80,8 +80,8 @@ public function findFromPersons(int $personId1, int $personId2) { $qb = $this->db->getQueryBuilder(); $qb->select('r.id', 'r.face1', 'r.face2', 'r.state') ->from($this->getTableName(), 'r') - ->where('r.face1 IN (' . $sub1->getSQL() . ')') - ->orWhere('r.face2 IN (' . $sub2->getSQL() . ')') + ->where('((r.face1 IN (' . $sub1->getSQL() . ')) AND (r.face2 IN (' . $sub2->getSQL() . ')))') + ->orWhere('((r.face2 IN (' . $sub1->getSQL() . ')) AND (r.face1 IN (' . $sub2->getSQL() . ')))') ->setParameter('person1', $personId1) ->setParameter('person2', $personId2); From d95067e2249a4d394173d80c3806233a5256b9a5 Mon Sep 17 00:00:00 2001 From: Matias De lellis Date: Wed, 22 Apr 2020 13:48:27 -0300 Subject: [PATCH 09/16] Also search inverted condition when find relations --- lib/Controller/PersonController.php | 2 +- lib/Db/RelationMapper.php | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/Controller/PersonController.php b/lib/Controller/PersonController.php index 2fab81c6..3747e6f8 100644 --- a/lib/Controller/PersonController.php +++ b/lib/Controller/PersonController.php @@ -221,7 +221,7 @@ public function findByName(string $personName) { * @param string $name */ public function updateName($id, $name) { - $person = $this->personMapper->find ($this->userId, $id); + $person = $this->personMapper->find($this->userId, $id); $person->setName($name); $this->personMapper->update($person); diff --git a/lib/Db/RelationMapper.php b/lib/Db/RelationMapper.php index 0079b3aa..09cf27a4 100644 --- a/lib/Db/RelationMapper.php +++ b/lib/Db/RelationMapper.php @@ -41,8 +41,8 @@ public function exists(Relation $relation): bool { $query = $qb ->select(['id']) ->from($this->getTableName()) - ->where($qb->expr()->eq('face1', $qb->createParameter('face1'))) - ->andWhere($qb->expr()->eq('face2', $qb->createParameter('face2'))) + ->where($qb->expr()->andX($qb->expr()->eq('face1', $qb->createParameter('face1')), $qb->expr()->eq('face2', $qb->createParameter('face2')))) + ->orWhere($qb->expr()->andX($qb->expr()->eq('face2', $qb->createParameter('face1')), $qb->expr()->eq('face1', $qb->createParameter('face2')))) ->setParameter('face1', $relation->getFace1()) ->setParameter('face2', $relation->getFace2()); From db8a6729922da622cc6ed021bf15cd2380bd0cc6 Mon Sep 17 00:00:00 2001 From: Matias De lellis Date: Wed, 22 Apr 2020 21:14:49 -0300 Subject: [PATCH 10/16] Some changes to optimize the filling of relations and print some info about it --- .../Tasks/CreateClustersTask.php | 25 +++++++------ lib/Db/FaceMapper.php | 37 +++++++++---------- lib/Db/RelationMapper.php | 16 ++++++++ 3 files changed, 48 insertions(+), 30 deletions(-) diff --git a/lib/BackgroundJob/Tasks/CreateClustersTask.php b/lib/BackgroundJob/Tasks/CreateClustersTask.php index 884df28e..5e9215a3 100644 --- a/lib/BackgroundJob/Tasks/CreateClustersTask.php +++ b/lib/BackgroundJob/Tasks/CreateClustersTask.php @@ -216,7 +216,8 @@ private function createClusterIfNeeded(string $userId) { } // Fill relation table with new clusters. - $this->fillFaceRelationsFromPersons($userId); + $relations = $this->fillFaceRelationsFromPersons($userId, $modelId); + $this->logInfo($relations . ' relations added as suggestions'); // Prevents not create/recreate the clusters unnecessarily. $this->settingsService->setNeedRecreateClusters(false, $userId); @@ -354,25 +355,26 @@ public function mergeClusters(array $oldCluster, array $newCluster): array { return $result; } - private function fillFaceRelationsFromPersons(string $userId) { + private function fillFaceRelationsFromPersons(string $userId, int $modelId): int { $deviation = $this->settingsService->getDeviation(); if (!version_compare(phpversion('pdlib'), '1.0.2', '>=') || ($deviation === 0.0)) - return; + return 0; $sensitivity = $this->settingsService->getSensitivity(); - $modelId = $this->settingsService->getCurrentFaceModel(); // Get the representative faces of each person $mainFaces = array(); $persons = $this->personMapper->findAll($userId, $modelId); foreach ($persons as $person) { - $mainFaces[] = $this->faceMapper->findRepresentativeFromPerson($userId, $person->getId(), $sensitivity, $modelId); + $mainFaces[] = $this->faceMapper->findRepresentativeFromPerson($userId, $modelId, $person->getId(), $sensitivity); } - // Get similar faces taking into account the deviation and insert new relations - for ($i = 0, $face_count1 = count($mainFaces); $i < $face_count1; $i++) { + // Get similar faces taking into account the deviation + $relations = array(); + $faces_count = count($mainFaces); + for ($i = 0 ; $i < $faces_count; $i++) { $face1 = $mainFaces[$i]; - for ($j = $i+1, $face_count2 = count($mainFaces); $j < $face_count2; $j++) { + for ($j = $i+1; $j < $faces_count; $j++) { $face2 = $mainFaces[$j]; $distance = dlib_vector_length($face1->descriptor, $face2->descriptor); if ($distance < ($sensitivity + $deviation)) { @@ -380,12 +382,13 @@ private function fillFaceRelationsFromPersons(string $userId) { $relation->setFace1($face1->getId()); $relation->setFace2($face2->getId()); $relation->setState(RELATION::PROPOSED); - if (!$this->relationMapper->exists($relation)) { - $this->relationMapper->insert($relation); - } + $relations[] = $relation; } } } + + // Merge new suggested relations + return $this->relationMapper->merge($relations); } } diff --git a/lib/Db/FaceMapper.php b/lib/Db/FaceMapper.php index c7fa9223..f37514c3 100644 --- a/lib/Db/FaceMapper.php +++ b/lib/Db/FaceMapper.php @@ -131,49 +131,48 @@ public function findFacesFromPerson(string $userId, int $personId, int $model, $ return $faces; } - public function findRepresentativeFromPerson(string $userId, int $personId, float $sensitivity, int $model) { - if (!version_compare(phpversion('pdlib'), '1.0.2', '>=')) { - return null; - } + public function findRepresentativeFromPerson(string $userId, int $modelId, int $personId, float $sensitivity) { $qb = $this->db->getQueryBuilder(); - $qb->select('f.id', 'f.image', 'f.person', 'f.descriptor') + $qb->select('f.id', 'f.descriptor') ->from($this->getTableName(), 'f') ->innerJoin('f', 'facerecog_images' ,'i', $qb->expr()->eq('f.image', 'i.id')) ->where($qb->expr()->eq('user', $qb->createNamedParameter($userId))) ->andWhere($qb->expr()->eq('person', $qb->createNamedParameter($personId))) - ->andWhere($qb->expr()->eq('model', $qb->createNamedParameter($model))); + ->andWhere($qb->expr()->eq('model', $qb->createNamedParameter($modelId))); $faces = $this->findEntities($qb); - $facesCount = array(); - for ($i = 0, $face_count1 = count($faces); $i < $face_count1; $i++) { + $face_count = count($faces); + $edgesFacesCount = array(); + for ($i = 0; $i < $face_count; $i++) { $face1 = $faces[$i]; - for ($j = $i, $face_count2 = count($faces); $j < $face_count2; $j++) { + for ($j = $i; $j < $face_count; $j++) { $face2 = $faces[$j]; $distance = dlib_vector_length($face1->descriptor, $face2->descriptor); if ($distance < $sensitivity) { - if (!array_key_exists($i, $facesCount)) { - $facesCount[$i] = 1; + if (isset($edgesFacesCount[$i])) { + $edgesFacesCount[$i]++; } else { - $facesCount[$i] = $facesCount[$i]+1; + $edgesFacesCount[$i] = 1; } - if (!array_key_exists($j, $facesCount)) { - $facesCount[$j] = 1; + if (isset($edgesFacesCount[$j])) { + $edgesFacesCount[$j]++; } else { - $facesCount[$j] = $facesCount[$j]+1; + $edgesFacesCount[$j] = 1; } } } } - $bestFaceId = -1; + $bestFaceIndex = -1; $bestFaceCount = -1; - foreach ($facesCount as $faceId => $faceCount) { + foreach ($edgesFacesCount as $faceIndex => $faceCount) { if ($faceCount > $bestFaceCount) { - $bestFaceId = $faceId; + $bestFaceIndex = $faceIndex; $bestFaceCount = $faceCount; } } - return $faces[$bestFaceId]; + + return $faces[$bestFaceIndex]; } public function getPersonOnFile(string $userId, int $personId, int $fileId, int $model): array { diff --git a/lib/Db/RelationMapper.php b/lib/Db/RelationMapper.php index 09cf27a4..e78d7d89 100644 --- a/lib/Db/RelationMapper.php +++ b/lib/Db/RelationMapper.php @@ -88,4 +88,20 @@ public function findFromPersons(int $personId1, int $personId2) { return $this->findEntities($qb); } + public function merge(array $relations): int { + $addedCount = 0; + + $this->db->beginTransaction(); + foreach ($relations as $relation) { + if ($this->exists($relation)) + continue; + + $this->insert($relation); + $addedCount++; + } + $this->db->commit(); + + return $addedCount; + } + } \ No newline at end of file From e726cff7297a4db9cafdef8088101d224545fb6c Mon Sep 17 00:00:00 2001 From: Matias De lellis Date: Fri, 24 Apr 2020 19:10:18 -0300 Subject: [PATCH 11/16] Use array as NxN array to optimize search for relations. Drop relations when resetting clusters --- .../Tasks/CreateClustersTask.php | 2 +- lib/Db/RelationMapper.php | 115 ++++++++++++++---- lib/Service/FaceManagementService.php | 7 ++ 3 files changed, 100 insertions(+), 24 deletions(-) diff --git a/lib/BackgroundJob/Tasks/CreateClustersTask.php b/lib/BackgroundJob/Tasks/CreateClustersTask.php index 5e9215a3..1b98036c 100644 --- a/lib/BackgroundJob/Tasks/CreateClustersTask.php +++ b/lib/BackgroundJob/Tasks/CreateClustersTask.php @@ -388,7 +388,7 @@ private function fillFaceRelationsFromPersons(string $userId, int $modelId): int } // Merge new suggested relations - return $this->relationMapper->merge($relations); + return $this->relationMapper->merge($userId, $modelId, $relations); } } diff --git a/lib/Db/RelationMapper.php b/lib/Db/RelationMapper.php index e78d7d89..b0b5fcd1 100644 --- a/lib/Db/RelationMapper.php +++ b/lib/Db/RelationMapper.php @@ -36,21 +36,25 @@ public function __construct(IDBConnection $db) { parent::__construct($db, 'facerecog_relations', '\OCA\FaceRecognition\Db\Relation'); } - public function exists(Relation $relation): bool { + /** + * Find all relation from that user. + * + * @param string $userId User user to search + * @param int $modelId + * @return array + */ + public function findByUser(string $userId, int $modelId): array { $qb = $this->db->getQueryBuilder(); - $query = $qb - ->select(['id']) - ->from($this->getTableName()) - ->where($qb->expr()->andX($qb->expr()->eq('face1', $qb->createParameter('face1')), $qb->expr()->eq('face2', $qb->createParameter('face2')))) - ->orWhere($qb->expr()->andX($qb->expr()->eq('face2', $qb->createParameter('face1')), $qb->expr()->eq('face1', $qb->createParameter('face2')))) - ->setParameter('face1', $relation->getFace1()) - ->setParameter('face2', $relation->getFace2()); - - $resultStatement = $query->execute(); - $row = $resultStatement->fetch(); - $resultStatement->closeCursor(); - - return ($row !== false); + $qb->select('r.id', 'r.face1', 'r.face2', 'r.state') + ->from($this->getTableName(), 'r') + ->innerJoin('r', 'facerecog_faces', 'f', $qb->expr()->eq('r.face1', 'f.id')) + ->innerJoin('f', 'facerecog_images', 'i', $qb->expr()->eq('f.image', 'i.id')) + ->where($qb->expr()->eq('i.user', $qb->createParameter('user_id'))) + ->andWhere($qb->expr()->eq('i.model', $qb->createParameter('model_id'))) + ->setParameter('user_id', $userId) + ->setParameter('model_id', $modelId); + + return $this->findEntities($qb); } public function findFromPerson(string $userId, int $personId, int $state): array { @@ -88,20 +92,85 @@ public function findFromPersons(int $personId1, int $personId2) { return $this->findEntities($qb); } - public function merge(array $relations): int { - $addedCount = 0; + /** + * Deletes all relations from that user. + * + * @param string $userId User to drop persons from a table. + */ + public function deleteUser(string $userId) { + $sub = $this->db->getQueryBuilder(); + $sub->select(new Literal('1')) + ->from('facerecog_faces', 'f') + ->innerJoin('f', 'facerecog_images', 'i', $sub->expr()->eq('f.image', 'i.id')) + ->andWhere($sub->expr()->eq('i.user', $sub->createParameter('user_id'))); - $this->db->beginTransaction(); + $qb = $this->db->getQueryBuilder(); + $qb->delete($this->getTableName()) + ->where('EXISTS (' . $sub->getSQL() . ')') + ->setParameter('user_id', $userId) + ->execute(); + } + + /** + * Find all the relations of a user as an matrix array, which is faster to access. + * @param string $userId + * @param int $modelId + * return array + */ + public function findByUserAsMatrix(string $userId, int $modelId): array { + $matrix = array(); + $relations = $this->findByUser($userId, $modelId); foreach ($relations as $relation) { - if ($this->exists($relation)) - continue; + $face1 = $relation->getFace1(); + $face2 = $relation->getFace2(); + $state = $relation->getState(); + + $row = array(); + if (isset($matrix[$face1])) { + $row = $matrix[$face1]; + } + $row[$face2] = $state; + $matrix[$face1] = $row; + } + return $matrix; + } - $this->insert($relation); - $addedCount++; + public function existsOnMatrix(Relation $relation, array $matrix): bool { + $face1 = $relation->getFace1(); + $face2 = $relation->getFace2(); + + if (isset($matrix[$face1])) { + $row = $matrix[$face1]; + if (isset($row[$face2])) { + return true; + } + } + if (isset($matrix[$face2])) { + $row = $matrix[$face2]; + if (isset($row[$face1])) { + return true; + } } - $this->db->commit(); + return false; + } - return $addedCount; + public function merge(string $userId, int $modelId, array $relations): int { + $added = 0; + $this->db->beginTransaction(); + try { + $oldMatrix = $this->findByUserAsMatrix($userId, $modelId); + foreach ($relations as $relation) { + if ($this->existsOnMatrix($relation, $oldMatrix)) + continue; + $this->insert($relation); + $added++; + } + $this->db->commit(); + } catch (\Exception $e) { + $this->db->rollBack(); + throw $e; + } + return $added; } } \ No newline at end of file diff --git a/lib/Service/FaceManagementService.php b/lib/Service/FaceManagementService.php index 1caee95f..8c25f29a 100644 --- a/lib/Service/FaceManagementService.php +++ b/lib/Service/FaceManagementService.php @@ -30,6 +30,7 @@ use OCA\FaceRecognition\Db\FaceMapper; use OCA\FaceRecognition\Db\ImageMapper; use OCA\FaceRecognition\Db\PersonMapper; +use OCA\FaceRecognition\Db\RelationMapper; use OCA\FaceRecognition\Service\SettingsService; @@ -56,6 +57,9 @@ class FaceManagementService { /** @var PersonMapper */ private $personMapper; + /** @var RelationMapper */ + private $relationMapper; + /** @var SettingsService */ private $settingsService; @@ -63,12 +67,14 @@ public function __construct(IUserManager $userManager, FaceMapper $faceMapper, ImageMapper $imageMapper, PersonMapper $personMapper, + RelationMapper $relationMapper, SettingsService $settingsService) { $this->userManager = $userManager; $this->faceMapper = $faceMapper; $this->imageMapper = $imageMapper; $this->personMapper = $personMapper; + $this->relationMapper = $relationMapper; $this->settingsService = $settingsService; } @@ -134,6 +140,7 @@ public function resetClustersForUser(string $userId) { $this->faceMapper->unsetPersonsRelationForUser($userId, $model); $this->personMapper->deleteUserPersons($userId); + $this->relationMapper->deleteUser($userId); } /** From a578af5f4a4ca1d3d5441118c9ca664c7a7afdc1 Mon Sep 17 00:00:00 2001 From: Matias De lellis Date: Fri, 24 Apr 2020 20:55:59 -0300 Subject: [PATCH 12/16] Proof of concept on improving clustering according to relation. --- .../Tasks/CreateClustersTask.php | 15 +++++++-- lib/Controller/RelationController.php | 4 +-- lib/Db/Relation.php | 6 ++-- lib/Db/RelationMapper.php | 33 ++++++++++++------- 4 files changed, 39 insertions(+), 19 deletions(-) diff --git a/lib/BackgroundJob/Tasks/CreateClustersTask.php b/lib/BackgroundJob/Tasks/CreateClustersTask.php index 1b98036c..bf66d394 100644 --- a/lib/BackgroundJob/Tasks/CreateClustersTask.php +++ b/lib/BackgroundJob/Tasks/CreateClustersTask.php @@ -196,12 +196,14 @@ private function createClusterIfNeeded(string $userId) { $faces = $this->faceMapper->getFaces($userId, $modelId); $this->logInfo(count($faces) . ' faces found for clustering'); + $relations = $this->relationMapper->findByUserAsMatrix($userId, $modelId); + // Cluster is associative array where key is person ID. // Value is array of face IDs. For old clusters, person IDs are some existing person IDs, // and for new clusters is whatever chinese whispers decides to identify them. // $currentClusters = $this->getCurrentClusters($faces); - $newClusters = $this->getNewClusters($faces); + $newClusters = $this->getNewClusters($faces, $relations); $this->logInfo(count($newClusters) . ' persons found after clustering'); // New merge @@ -237,7 +239,7 @@ private function getCurrentClusters(array $faces): array { return $chineseClusters; } - private function getNewClusters(array $faces): array { + private function getNewClusters(array $faces, array $relations): array { // Create edges for chinese whispers $sensitivity = $this->settingsService->getSensitivity(); $min_confidence = $this->settingsService->getMinimumConfidence(); @@ -252,6 +254,15 @@ private function getNewClusters(array $faces): array { } for ($j = $i, $face_count2 = count($faces); $j < $face_count2; $j++) { $face2 = $faces[$j]; + if ($this->relationMapper->existsOnMatrix($face1->id, $face2->id, $relations)) { + $state = $this->relationMapper->getStateOnMatrix($face1->id, $face2->id, $relations); + if ($state === Relation::ACCEPTED) { + $edges[] = array($i, $j); + continue; + } else if ($state === Relation::REJECTED) { + continue; + } + } $distance = dlib_vector_length($face1->descriptor, $face2->descriptor); if ($distance < $sensitivity) { diff --git a/lib/Controller/RelationController.php b/lib/Controller/RelationController.php index e407b25d..339fbf8b 100644 --- a/lib/Controller/RelationController.php +++ b/lib/Controller/RelationController.php @@ -89,7 +89,7 @@ public function findByPerson(int $personId) { $proposed = array(); $relations = $this->relationMapper->findFromPerson($this->userId, $personId, RELATION::PROPOSED); foreach ($relations as $relation) { - $person1 = $this->personMapper->findFromFace($this->userId, $relation->getFace1()); + $person1 = $this->personMapper->findFromFace($this->userId, $relation->face1); if (($person1->getId() !== $personId) && ($mainPerson->getName() !== $person1->getName())) { $proffer = array(); $proffer['origId'] = $mainPerson->getId(); @@ -97,7 +97,7 @@ public function findByPerson(int $personId) { $proffer['name'] = $person1->getName(); $proposed[] = $proffer; } - $person2 = $this->personMapper->findFromFace($this->userId, $relation->getFace2()); + $person2 = $this->personMapper->findFromFace($this->userId, $relation->face2); if (($person2->getId() !== $personId) && ($mainPerson->getName() !== $person2->getName())) { $proffer = array(); $proffer['origId'] = $mainPerson->getId(); diff --git a/lib/Db/Relation.php b/lib/Db/Relation.php index b2817336..c13e208a 100644 --- a/lib/Db/Relation.php +++ b/lib/Db/Relation.php @@ -48,14 +48,14 @@ class Relation extends Entity { * * @var int * */ - protected $face1; + public $face1; /** * Face id of a face of a person related with $face1 * * @var int * */ - protected $face2; + public $face2; /** * State of two face relation. These are proposed, and can be accepted @@ -63,6 +63,6 @@ class Relation extends Entity { * * @var int * */ - protected $state; + public $state; } \ No newline at end of file diff --git a/lib/Db/RelationMapper.php b/lib/Db/RelationMapper.php index b0b5fcd1..efa67acc 100644 --- a/lib/Db/RelationMapper.php +++ b/lib/Db/RelationMapper.php @@ -121,24 +121,33 @@ public function findByUserAsMatrix(string $userId, int $modelId): array { $matrix = array(); $relations = $this->findByUser($userId, $modelId); foreach ($relations as $relation) { - $face1 = $relation->getFace1(); - $face2 = $relation->getFace2(); - $state = $relation->getState(); - $row = array(); - if (isset($matrix[$face1])) { - $row = $matrix[$face1]; + if (isset($matrix[$relation->face1])) { + $row = $matrix[$relation->face1]; } - $row[$face2] = $state; - $matrix[$face1] = $row; + $row[$relation->face2] = $relation->state; + $matrix[$relation->face1] = $row; } return $matrix; } - public function existsOnMatrix(Relation $relation, array $matrix): bool { - $face1 = $relation->getFace1(); - $face2 = $relation->getFace2(); + public function getStateOnMatrix(int $face1, int $face2, array $matrix): int { + if (isset($matrix[$face1])) { + $row = $matrix[$face1]; + if (isset($row[$face2])) { + return $matrix[$face1][$face2]; + } + } + if (isset($matrix[$face2])) { + $row = $matrix[$face2]; + if (isset($row[$face1])) { + return $matrix[$face2][$face1]; + } + } + return Relation::PROPOSED; + } + public function existsOnMatrix(int $face1, int $face2, array $matrix): bool { if (isset($matrix[$face1])) { $row = $matrix[$face1]; if (isset($row[$face2])) { @@ -160,7 +169,7 @@ public function merge(string $userId, int $modelId, array $relations): int { try { $oldMatrix = $this->findByUserAsMatrix($userId, $modelId); foreach ($relations as $relation) { - if ($this->existsOnMatrix($relation, $oldMatrix)) + if ($this->existsOnMatrix($relation->face1, $relation->face2, $oldMatrix)) continue; $this->insert($relation); $added++; From f7a85f9d1b123d09095c67875969f04638b78a74 Mon Sep 17 00:00:00 2001 From: Matias De lellis Date: Fri, 24 Apr 2020 22:22:17 -0300 Subject: [PATCH 13/16] Doh. Also don't compare a good face with a bad one! --- lib/BackgroundJob/Tasks/CreateClustersTask.php | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/BackgroundJob/Tasks/CreateClustersTask.php b/lib/BackgroundJob/Tasks/CreateClustersTask.php index bf66d394..a8ba8dc7 100644 --- a/lib/BackgroundJob/Tasks/CreateClustersTask.php +++ b/lib/BackgroundJob/Tasks/CreateClustersTask.php @@ -263,8 +263,10 @@ private function getNewClusters(array $faces, array $relations): array { continue; } } + if ($face2->confidence < $min_confidence) { + continue; + } $distance = dlib_vector_length($face1->descriptor, $face2->descriptor); - if ($distance < $sensitivity) { $edges[] = array($i, $j); } @@ -280,6 +282,9 @@ private function getNewClusters(array $faces, array $relations): array { } for ($j = $i, $face_count2 = count($faces); $j < $face_count2; $j++) { $face2 = $faces[$j]; + if ($face2->confidence < $min_confidence) { + continue; + } // todo: can't this distance be a method in $face1->distance($face2)? $distance = $euclidean->distance($face1->descriptor, $face2->descriptor); From 776de523ce0f53a51ae46558edfffdfca77d5f35 Mon Sep 17 00:00:00 2001 From: Matias De lellis Date: Sun, 26 Apr 2020 00:31:39 -0300 Subject: [PATCH 14/16] Suggest all proposals, even if they already have the same name. Then filter (Auto accepting or rejecting the proposals) while you accept the proposals. This ensures that you can apply as many relations as possible, but must accept fewer suggestions. On the other hand, now not apply any change until finish with all the suggestions. So, if you close the dialog no changes will be applied. --- appinfo/routes.php | 6 ++ js/personal.js | 58 +++++++----- js/similar.js | 88 ++++++++++++++++--- .../Tasks/CreateClustersTask.php | 9 +- lib/Controller/RelationController.php | 34 ++++++- 5 files changed, 155 insertions(+), 40 deletions(-) diff --git a/appinfo/routes.php b/appinfo/routes.php index 4d402f9d..cd3efd3c 100644 --- a/appinfo/routes.php +++ b/appinfo/routes.php @@ -55,6 +55,12 @@ 'url' => '/folder', 'verb' => 'GET' ], + // Change relations in batch + [ + 'name' => 'relation#updateByPersonsBatch', + 'url' => '/relations', + 'verb' => 'PUT' + ], // Set folder preferences [ 'name' => 'file#setFolderOptions', diff --git a/js/personal.js b/js/personal.js index 6dc77708..e6fb45d5 100644 --- a/js/personal.js +++ b/js/personal.js @@ -30,6 +30,7 @@ Persons.prototype = { self._enabled = response.enabled; self._clusters = response.clusters; self._loaded = true; + console.debug('The user has %i clusters',self._clusters.length); deferred.resolve(); }).fail(function () { deferred.reject(); @@ -43,6 +44,7 @@ Persons.prototype = { var self = this; $.get(this._baseUrl+'/cluster/'+id).done(function (cluster) { self._cluster = cluster; + console.debug('Cluster id %i has %i faces', id, cluster.faces.length); deferred.resolve(); }).fail(function () { deferred.reject(); @@ -54,6 +56,11 @@ Persons.prototype = { var self = this; $.get(this._baseUrl+'/person/'+personName).done(function (clusters) { self._clustersByName = clusters.clusters; + var total = 0; + self._clustersByName.forEach(function (cluster) { + total += cluster.faces.length; + }); + console.debug('There are %i clusters called %s with a total of %i faces', self._clustersByName.length, personName, total); deferred.resolve(); }).fail(function () { deferred.reject(); @@ -194,33 +201,40 @@ View.prototype = { function(valid, state) { if (valid === true) { // It is valid must be update the proposals. - self._similar.updateProposal(proposal, state).done(function() { - if (state === Relation.ACCEPTED) { - // Update view with new name. - self._persons.renameCluster(proposal.id, personName); - self._persons.unsetActive(); - self.renderContent(); - // Look for new suggestions based on accepted proposal - self._similar.findProposal(proposal.id, personName).done(function() { - if (self._similar.hasProposal()) { - self.suggestPerson(self._similar.getProposal(), personName); - } else { - OC.Notification.showTemporary(t('facerecognition', 'There are no more suggestions from similar persons')); - } - }); - } else { - // Suggest cached proposals + self._similar.answerProposal(proposal, state); + if (state === Relation.ACCEPTED) { + // Look for new suggestions based on accepted proposal + self._similar.findProposal(proposal.id, personName).done(function() { if (self._similar.hasProposal()) { self.suggestPerson(self._similar.getProposal(), personName); } else { - OC.Notification.showTemporary(t('facerecognition', 'There are no more suggestions from similar persons')); + OC.Notification.showTemporary(t('facerecognition', 'There are no more suggestions. Applying your suggestions.')); + self._similar.applyProposals().done(function() { + self._similar.getAcceptedProposal().forEach(function (accepted) { + self._persons.renameCluster(accepted.id, personName); + }); + self._persons.unsetActive(); + self.renderContent(); + }); } + }); + } else { + // Suggest cached proposals + if (self._similar.hasProposal()) { + self.suggestPerson(self._similar.getProposal(), personName); + } else { + OC.Notification.showTemporary(t('facerecognition', 'There are no more suggestions. Applying your suggestions.')); + self._similar.applyProposals().done(function() { + self._similar.getAcceptedProposal().forEach(function (accepted) { + self._persons.renameCluster(accepted.id, personName); + }); + self._persons.unsetActive(); + self.renderContent(); + }); } - }).fail(function () { - if (state === Relation.ACCEPTED) { - OC.Notification.showTemporary(t('facerecognition', 'There was an error renaming this person')); - } - }); + } + } else { + OC.Notification.showTemporary(t('facerecognition', 'Canceled')); } } ); diff --git a/js/similar.js b/js/similar.js index 5a1362fc..ab48911e 100644 --- a/js/similar.js +++ b/js/similar.js @@ -11,7 +11,10 @@ var Similar = function (baseUrl) { this._enabled = false; this._similarProposed = []; + this._similarAccepted = []; this._similarRejected = []; + this._similarIgnored = []; + this._similarApplied = []; this._similarName = undefined; }; @@ -21,14 +24,14 @@ Similar.prototype = { }, findProposal: function (clusterId, clusterName) { if (this._similarName !== clusterName) { - this.resetProposals(); + this.resetProposals(clusterId, clusterName); } var self = this; var deferred = $.Deferred(); $.get(this._baseUrl+'/relation/'+clusterId).done(function (response) { self._enabled = response.enabled; if (!self._enabled) { - self.resetSuggestions(); + self.resetSuggestions(clusterId, clusterName); } else { self.concatNewProposals(response.proposed); self._similarName = clusterName; @@ -45,21 +48,58 @@ Similar.prototype = { getProposal: function () { return this._similarProposed.shift(); }, - updateProposal: function (proposal, newState) { + getAcceptedProposal: function () { + return this._similarAccepted; + }, + answerProposal: function (proposal, state) { + var self = this; + // First of all, remove from queue all relations with that ID + var newProposed = []; + self._similarProposed.forEach(function (oldProposal) { + if (proposal.id !== oldProposal.id) { + newProposed.push(oldProposal); + } else { + if (state === Relation.ACCEPTED) { + oldProposal.state = Relation.ACCEPTED; + self._similarAccepted.push(oldProposal); + self._similarApplied.push(oldProposal); + } else if (state === Relation.REJECTED) { + oldProposal.state = Relation.REJECTED; + self._similarRejected.push(oldProposal); + self._similarApplied.push(oldProposal); + } else { + oldProposal.state = Relation.PROPOSED; + self._similarIgnored.push(oldProposal); + } + } + }); + self._similarProposed = newProposed; + + // Add the old proposal to its actual state. + proposal.state = state; + if (state === Relation.ACCEPTED) { + this._similarAccepted.push(proposal); + this._similarApplied.push(proposal); + } else if (state === Relation.REJECTED) { + this._similarRejected.push(proposal); + this._similarApplied.push(proposal); + } else { + this._similarIgnored.push(proposal); + } + }, + applyProposals: function () { var self = this; var deferred = $.Deferred(); var data = { - toPersonId: proposal.id, - state: newState + personsRelations: self._similarApplied, + personName: self._similarName }; $.ajax({ - url: this._baseUrl + '/relation/' + proposal.origId, + url: this._baseUrl + '/relations', method: 'PUT', contentType: 'application/json', data: JSON.stringify(data) }).done(function (data) { - if (newState !== Relation.ACCEPTED) - self._similarRejected.push(proposal); deferred.resolve(); }).fail(function () { deferred.reject(); @@ -69,15 +109,39 @@ Similar.prototype = { concatNewProposals: function (proposals) { var self = this; proposals.forEach(function (proposed) { - if ((self._similarProposed.find(function (oldProposed) { return proposed.id === oldProposed.id;}) === undefined) && - (self._similarRejected.find(function (rejProposed) { return proposed.id === rejProposed.id;}) === undefined)) { + // An person ca be propoced several times, since they are related to direfents persons. + if (self._similarAccepted.find(function (oldProposed) { return proposed.id === oldProposed.id;}) !== undefined) { + proposed.state = Relation.ACCEPTED; + self._similarAccepted.push(proposed); + self._similarApplied.push(proposed); + } else if (self._similarRejected.find(function (oldProposed) { return proposed.id === oldProposed.id;}) !== undefined) { + proposed.state = Relation.REJECTED; + self._similarRejected.push(proposed); + self._similarApplied.push(proposed); + } else if (self._similarIgnored.find(function (oldProposed) { return proposed.id === oldProposed.id;}) !== undefined) { + proposed.state = Relation.REJECTED; + self._similarIgnored.push(proposed); + } else { + proposed.state = Relation.PROPOSED; self._similarProposed.push(proposed); } }); }, - resetProposals: function () { + resetProposals: function (clusterId, clusterName) { + this._similarAccepted = []; this._similarProposed = []; this._similarRejected = []; + this._similarIgnored = []; + this._similarApplied = []; this._similarName = undefined; - }, + + // Add a fake proposal to self-accept when referring to the initial person + var fakeProposal = { + origId: clusterId, + id: clusterId, + name: clusterName, + state: Relation.ACCEPTED + }; + this._similarAccepted.push(fakeProposal); + } }; \ No newline at end of file diff --git a/lib/BackgroundJob/Tasks/CreateClustersTask.php b/lib/BackgroundJob/Tasks/CreateClustersTask.php index a8ba8dc7..bf8e0489 100644 --- a/lib/BackgroundJob/Tasks/CreateClustersTask.php +++ b/lib/BackgroundJob/Tasks/CreateClustersTask.php @@ -377,6 +377,7 @@ private function fillFaceRelationsFromPersons(string $userId, int $modelId): int return 0; $sensitivity = $this->settingsService->getSensitivity(); + $sensitivity += $deviation; // Get the representative faces of each person $mainFaces = array(); @@ -393,11 +394,11 @@ private function fillFaceRelationsFromPersons(string $userId, int $modelId): int for ($j = $i+1; $j < $faces_count; $j++) { $face2 = $mainFaces[$j]; $distance = dlib_vector_length($face1->descriptor, $face2->descriptor); - if ($distance < ($sensitivity + $deviation)) { + if ($distance < $sensitivity) { $relation = new Relation(); - $relation->setFace1($face1->getId()); - $relation->setFace2($face2->getId()); - $relation->setState(RELATION::PROPOSED); + $relation->setFace1($face1->id); + $relation->setFace2($face2->id); + $relation->setState(Relation::PROPOSED); $relations[] = $relation; } } diff --git a/lib/Controller/RelationController.php b/lib/Controller/RelationController.php index 339fbf8b..c2e59573 100644 --- a/lib/Controller/RelationController.php +++ b/lib/Controller/RelationController.php @@ -90,7 +90,7 @@ public function findByPerson(int $personId) { $relations = $this->relationMapper->findFromPerson($this->userId, $personId, RELATION::PROPOSED); foreach ($relations as $relation) { $person1 = $this->personMapper->findFromFace($this->userId, $relation->face1); - if (($person1->getId() !== $personId) && ($mainPerson->getName() !== $person1->getName())) { + if ($person1->getId() !== $personId) { $proffer = array(); $proffer['origId'] = $mainPerson->getId(); $proffer['id'] = $person1->getId(); @@ -98,7 +98,7 @@ public function findByPerson(int $personId) { $proposed[] = $proffer; } $person2 = $this->personMapper->findFromFace($this->userId, $relation->face2); - if (($person2->getId() !== $personId) && ($mainPerson->getName() !== $person2->getName())) { + if ($person2->getId() !== $personId) { $proffer = array(); $proffer['origId'] = $mainPerson->getId(); $proffer['id'] = $person2->getId(); @@ -138,4 +138,34 @@ public function updateByPersons(int $personId, int $toPersonId, int $state) { return new DataResponse($relations); } + /** + * @NoAdminRequired + * @param array $personsRelations + * @param string $personName + */ + public function updateByPersonsBatch(array $personsRelations, string $personName) { + foreach ($personsRelations as $personRelation) { + $origId = $personRelation['origId']; + $id = $personRelation['id']; + $state = $personRelation['state']; + + $faceRelations = $this->relationMapper->findFromPersons($origId, $id); + foreach ($faceRelations as $faceRelation) { + $faceRelation->setState($state); + $this->relationMapper->update($faceRelation); + } + + if ($state === RELATION::ACCEPTED) { + $toPerson = $this->personMapper->find($this->userId, $id); + $toPerson->setName($personName); + $this->personMapper->update($toPerson); + } + } + + $modelId = $this->settingsService->getCurrentFaceModel(); + $persons = $this->personMapper->findByName($this->userId, $modelId, $personName); + + return new DataResponse($persons); + } + } From 0b22b79031f5f8c87e27e1a535b1ee4b63005bec Mon Sep 17 00:00:00 2001 From: Matias De lellis Date: Tue, 28 Apr 2020 12:21:13 -0300 Subject: [PATCH 15/16] Also, dont suggest persons/faces with less than minimum confidence. --- lib/BackgroundJob/Tasks/CreateClustersTask.php | 10 +++++++++- lib/Db/FaceMapper.php | 8 ++++---- lib/Db/RelationMapper.php | 13 ++++++++----- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/lib/BackgroundJob/Tasks/CreateClustersTask.php b/lib/BackgroundJob/Tasks/CreateClustersTask.php index bf8e0489..c190f27e 100644 --- a/lib/BackgroundJob/Tasks/CreateClustersTask.php +++ b/lib/BackgroundJob/Tasks/CreateClustersTask.php @@ -379,11 +379,13 @@ private function fillFaceRelationsFromPersons(string $userId, int $modelId): int $sensitivity = $this->settingsService->getSensitivity(); $sensitivity += $deviation; + $min_confidence = $this->settingsService->getMinimumConfidence(); + // Get the representative faces of each person $mainFaces = array(); $persons = $this->personMapper->findAll($userId, $modelId); foreach ($persons as $person) { - $mainFaces[] = $this->faceMapper->findRepresentativeFromPerson($userId, $modelId, $person->getId(), $sensitivity); + $mainFaces[] = $this->faceMapper->findRepresentativeFromPerson($userId, $modelId, $person->id, $sensitivity); } // Get similar faces taking into account the deviation @@ -391,8 +393,14 @@ private function fillFaceRelationsFromPersons(string $userId, int $modelId): int $faces_count = count($mainFaces); for ($i = 0 ; $i < $faces_count; $i++) { $face1 = $mainFaces[$i]; + if ($face1->confidence < $min_confidence) { + continue; + } for ($j = $i+1; $j < $faces_count; $j++) { $face2 = $mainFaces[$j]; + if ($face2->confidence < $min_confidence) { + continue; + } $distance = dlib_vector_length($face1->descriptor, $face2->descriptor); if ($distance < $sensitivity) { $relation = new Relation(); diff --git a/lib/Db/FaceMapper.php b/lib/Db/FaceMapper.php index f37514c3..d11d5a47 100644 --- a/lib/Db/FaceMapper.php +++ b/lib/Db/FaceMapper.php @@ -40,7 +40,7 @@ public function find (int $faceId) { $qb = $this->db->getQueryBuilder(); $qb->select('id', 'image', 'person', 'left', 'right', 'top', 'bottom', 'landmarks', 'descriptor', 'confidence') ->from($this->getTableName(), 'f') - ->andWhere($qb->expr()->eq('id', $qb->createNamedParameter($faceId))); + ->where($qb->expr()->eq('id', $qb->createNamedParameter($faceId))); return $this->findEntity($qb); } @@ -103,7 +103,7 @@ public function getOldestCreatedFaceWithoutPerson(string $userId, int $model) { return $face; } - public function getFaces(string $userId, $model): array { + public function getFaces(string $userId, int $modelId): array { $qb = $this->db->getQueryBuilder(); $qb->select('f.id', 'f.person', 'f.confidence', 'f.descriptor') ->from($this->getTableName(), 'f') @@ -111,7 +111,7 @@ public function getFaces(string $userId, $model): array { ->where($qb->expr()->eq('user', $qb->createParameter('user'))) ->andWhere($qb->expr()->eq('model', $qb->createParameter('model'))) ->setParameter('user', $userId) - ->setParameter('model', $model); + ->setParameter('model', $modelId); return $this->findEntities($qb); } @@ -133,7 +133,7 @@ public function findFacesFromPerson(string $userId, int $personId, int $model, $ public function findRepresentativeFromPerson(string $userId, int $modelId, int $personId, float $sensitivity) { $qb = $this->db->getQueryBuilder(); - $qb->select('f.id', 'f.descriptor') + $qb->select('f.id', 'f.confidence', 'f.descriptor') ->from($this->getTableName(), 'f') ->innerJoin('f', 'facerecog_images' ,'i', $qb->expr()->eq('f.image', 'i.id')) ->where($qb->expr()->eq('user', $qb->createNamedParameter($userId))) diff --git a/lib/Db/RelationMapper.php b/lib/Db/RelationMapper.php index efa67acc..09d37ff9 100644 --- a/lib/Db/RelationMapper.php +++ b/lib/Db/RelationMapper.php @@ -58,14 +58,17 @@ public function findByUser(string $userId, int $modelId): array { } public function findFromPerson(string $userId, int $personId, int $state): array { + $sub = $this->db->getQueryBuilder(); + $sub->select('f.id') + ->from('facerecog_faces', 'f') + ->where($sub->expr()->eq('f.person', $sub->createParameter('person_id'))); + $qb = $this->db->getQueryBuilder(); $qb->select('r.id', 'r.face1', 'r.face2', 'r.state') ->from($this->getTableName(), 'r') - ->innerJoin('r', 'facerecog_faces' ,'f', $qb->expr()->orX($qb->expr()->eq('r.face1', 'f.id'), $qb->expr()->eq('r.face2', 'f.id'))) - ->innerJoin('f', 'facerecog_persons' ,'p', $qb->expr()->eq('f.person', 'p.id')) - ->where($qb->expr()->eq('p.user', $qb->createNamedParameter($userId))) - ->andWhere($qb->expr()->eq('p.id', $qb->createNamedParameter($personId))) - ->andWhere($qb->expr()->eq('r.state', $qb->createNamedParameter($state))); + ->where('(r.face1 IN (' . $sub->getSQL() . '))') + ->orWhere('(r.face2 IN (' . $sub->getSQL() . '))') + ->setParameter('person_id', $personId); return $this->findEntities($qb); } From 8189e17757bd435d3de19f172b9be69b871337ca Mon Sep 17 00:00:00 2001 From: Matias De lellis Date: Tue, 28 Apr 2020 13:13:00 -0300 Subject: [PATCH 16/16] Take into account state when find first relations --- lib/Db/RelationMapper.php | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/Db/RelationMapper.php b/lib/Db/RelationMapper.php index 09d37ff9..f99c36b0 100644 --- a/lib/Db/RelationMapper.php +++ b/lib/Db/RelationMapper.php @@ -61,14 +61,19 @@ public function findFromPerson(string $userId, int $personId, int $state): array $sub = $this->db->getQueryBuilder(); $sub->select('f.id') ->from('facerecog_faces', 'f') - ->where($sub->expr()->eq('f.person', $sub->createParameter('person_id'))); + ->innerJoin('f', 'facerecog_persons' ,'p', $sub->expr()->eq('f.person', 'p.id')) + ->where($sub->expr()->eq('p.user', $sub->createParameter('user_id'))) + ->andWhere($sub->expr()->eq('f.person', $sub->createParameter('person_id'))); $qb = $this->db->getQueryBuilder(); $qb->select('r.id', 'r.face1', 'r.face2', 'r.state') ->from($this->getTableName(), 'r') - ->where('(r.face1 IN (' . $sub->getSQL() . '))') + ->where($qb->expr()->eq('r.state', $qb->createParameter('state'))) + ->andWhere('(r.face1 IN (' . $sub->getSQL() . '))') ->orWhere('(r.face2 IN (' . $sub->getSQL() . '))') - ->setParameter('person_id', $personId); + ->setParameter('user_id', $userId) + ->setParameter('person_id', $personId) + ->setParameter('state', $state); return $this->findEntities($qb); }