feat(server): separate face clustering job (#5598)

* separate facial clustering job * update api * fixed some tests * invert clustering * hdbscan * update api * remove commented code * wip dbscan * cleanup removed cluster endpoint remove commented code * fixes updated tests minor fixes and formatting fixed queuing refinements * scale search range based on library size * defer non-core faces * optimizations removed unused query option * assign faces individually for correctness fixed unit tests remove unused method * don't select face embedding update sql linting fixed ml typing * updated job mock * paginate people query * select face embeddings because typeorm * fix setting face detection concurrency * update sql formatting linting * simplify logic remove unused imports * more specific delete signature * more accurate typing for face stubs * add migration formatting * chore: better typing * don't select embedding by default remove unused import * updated sql * use normal try/catch * stricter concurrency typing and enforcement * update api * update job concurrency panel to show disabled queues formatting * check jobId in queueAll fix tests * remove outdated comment * better facial recognition icon * wording wording formatting * fixed tests * fix * formatting & sql * try to fix sql check * more detailed description * update sql * formatting * wording * update `minFaces` description --------- Co-authored-by: Jason Rasmussen <jrasm91@gmail.com> Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2024-01-18 00:08:48 -05:00
parent 44873b4224
commit 68f52818ae
57 changed files with 1081 additions and 631 deletions
@@ -204,16 +204,20 @@ export class AuditService {
      }
    }

-    const people = await this.personRepository.getAll();
-    for (const { id, thumbnailPath } of people) {
-      track(thumbnailPath);
-      const entity = { entityId: id, entityType: PathEntityType.PERSON };
-      if (thumbnailPath && !hasFile(thumbFiles, thumbnailPath)) {
-        orphans.push({ ...entity, pathType: PersonPathType.FACE, pathValue: thumbnailPath });
+    const personPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
+      this.personRepository.getAll(pagination),
+    );
+    for await (const people of personPagination) {
+      for (const { id, thumbnailPath } of people) {
+        track(thumbnailPath);
+        const entity = { entityId: id, entityType: PathEntityType.PERSON };
+        if (thumbnailPath && !hasFile(thumbFiles, thumbnailPath)) {
+          orphans.push({ ...entity, pathType: PersonPathType.FACE, pathValue: thumbnailPath });
+        }
      }
-    }

-    this.logger.log(`Found ${assetCount} assets, ${users.length} users, ${people.length} people`);
+      this.logger.log(`Found ${assetCount} assets, ${users.length} users, ${people.length} people`);
+    }

    const extras: string[] = [];
    for (const file of allFiles) {