feat(server): near-duplicate detection (#8228)

* duplicate detection job, entity, config * queueing * job panel, update api * use embedding in db instead of fetching * disable concurrency * only queue visible assets * handle multiple duplicateIds * update concurrent queue check * add provider * add web placeholder, server endpoint, migration, various fixes * update sql * select embedding by default * rename variable * simplify * remove separate entity, handle re-running with different threshold, set default back to 0.02 * fix tests * add tests * add index to entity * formatting * update asset mock * fix `upsertJobStatus` signature * update sql * formatting * default to 0.03 * optimize clustering * use asset's `duplicateId` if present * update sql * update tests * expose admin setting * refactor * formatting * skip if ml is disabled * debug trash e2e * remove from web * remove from sidebar * test if ml is disabled * update sql * separate duplicate detection from clip in config, disable by default for now * fix doc * lower minimum `maxDistance` * update api * Add and Use Duplicate Detection Feature Flag (#9364) * Add Duplicate Detection Flag * Use Duplicate Detection Flag * Attempt Fixes for Failing Checks * lower minimum `maxDistance` * fix tests --------- Co-authored-by: mertalev <101130780+mertalev@users.noreply.github.com> * chore: fixes and additions after rebase * chore: update api (remove new Role enum) * fix: left join smart search so getAll works without machine learning * test: trash e2e go back to checking length of assets is zero * chore: regen api after rebase * test: fix tests after rebase * redundant join --------- Co-authored-by: Nicholas Flamy <30300649+NicholasFlamy@users.noreply.github.com> Co-authored-by: Zack Pollard <zackpollard@ymail.com> Co-authored-by: Zack Pollard <zack@futo.org>
2024-05-16 13:08:37 -04:00
parent 673e97e71d
commit 64636c0618
61 changed files with 1254 additions and 61 deletions
@@ -115,6 +115,10 @@ export class JobService {
        return this.jobRepository.queue({ name: JobName.QUEUE_SMART_SEARCH, data: { force } });
      }

+      case QueueName.DUPLICATE_DETECTION: {
+        return this.jobRepository.queue({ name: JobName.QUEUE_DUPLICATE_DETECTION, data: { force } });
+      }
+
      case QueueName.METADATA_EXTRACTION: {
        return this.jobRepository.queue({ name: JobName.QUEUE_METADATA_EXTRACTION, data: { force } });
      }
@@ -191,7 +195,11 @@ export class JobService {
  }

  private isConcurrentQueue(name: QueueName): name is ConcurrentQueueName {
-    return ![QueueName.FACIAL_RECOGNITION, QueueName.STORAGE_TEMPLATE_MIGRATION].includes(name);
+    return ![
+      QueueName.FACIAL_RECOGNITION,
+      QueueName.STORAGE_TEMPLATE_MIGRATION,
+      QueueName.DUPLICATE_DETECTION,
+    ].includes(name);
  }

  async handleNightlyJobs() {
@@ -294,6 +302,13 @@ export class JobService {
        break;
      }

+      case JobName.SMART_SEARCH: {
+        if (item.data.source === 'upload') {
+          await this.jobRepository.queue({ name: JobName.DUPLICATE_DETECTION, data: item.data });
+        }
+        break;
+      }
+
      case JobName.USER_DELETION: {
        this.eventRepository.clientBroadcast(ClientEvent.USER_DELETE, item.data.id);
        break;