fix(server): external library sync not working for large libraries (#7759)

server/src/infra/repositories/asset.repository.ts

@@ -2,6 +2,7 @@ import {
   AssetBuilderOptions,
   AssetCreate,
   AssetExploreFieldOptions,
+  AssetPathEntity,
   AssetSearchOptions,
   AssetStats,
   AssetStatsOptions,
@@ -184,10 +185,10 @@ export class AssetRepository implements IAssetRepository {
   }
 
   @GenerateSql({ params: [[DummyValue.UUID]] })
-  @ChunkedArray()
-  getByLibraryId(libraryIds: string[]): Promise<AssetEntity[]> {
-    return this.repository.find({
-      where: { library: { id: In(libraryIds) } },
+  getLibraryAssetPaths(pagination: PaginationOptions, libraryId: string): Paginated<AssetPathEntity> {
+    return paginate(this.repository, pagination, {
+      select: { id: true, originalPath: true, isOffline: true },
+      where: { library: { id: libraryId } },
     });
   }
 

server/src/infra/repositories/filesystem.provider.ts

@@ -11,7 +11,7 @@ import {
 import { ImmichLogger } from '@app/infra/logger';
 import archiver from 'archiver';
 import chokidar, { WatchOptions } from 'chokidar';
-import { glob } from 'glob';
+import { glob } from 'fast-glob';
 import { constants, createReadStream, existsSync, mkdirSync } from 'node:fs';
 import fs, { copyFile, readdir, rename, stat, utimes, writeFile } from 'node:fs/promises';
 import path from 'node:path';
@@ -123,7 +123,7 @@ export class FilesystemProvider implements IStorageRepository {
 
   crawl(crawlOptions: CrawlOptionsDto): Promise<string[]> {
     const { pathsToCrawl, exclusionPatterns, includeHidden } = crawlOptions;
-    if (!pathsToCrawl) {
+    if (pathsToCrawl.length === 0) {
       return Promise.resolve([]);
     }
 
@@ -132,8 +132,8 @@ export class FilesystemProvider implements IStorageRepository {
 
     return glob(`${base}/**/${extensions}`, {
       absolute: true,
-      nocase: true,
-      nodir: true,
+      caseSensitiveMatch: false,
+      onlyFiles: true,
       dot: includeHidden,
       ignore: exclusionPatterns,
     });

server/src/infra/sql/asset.repository.sql

@@ -293,53 +293,6 @@ DELETE FROM "assets"
 WHERE
   "ownerId" = $1
 
--- AssetRepository.getByLibraryId
-SELECT
-  "AssetEntity"."id" AS "AssetEntity_id",
-  "AssetEntity"."deviceAssetId" AS "AssetEntity_deviceAssetId",
-  "AssetEntity"."ownerId" AS "AssetEntity_ownerId",
-  "AssetEntity"."libraryId" AS "AssetEntity_libraryId",
-  "AssetEntity"."deviceId" AS "AssetEntity_deviceId",
-  "AssetEntity"."type" AS "AssetEntity_type",
-  "AssetEntity"."originalPath" AS "AssetEntity_originalPath",
-  "AssetEntity"."resizePath" AS "AssetEntity_resizePath",
-  "AssetEntity"."webpPath" AS "AssetEntity_webpPath",
-  "AssetEntity"."thumbhash" AS "AssetEntity_thumbhash",
-  "AssetEntity"."encodedVideoPath" AS "AssetEntity_encodedVideoPath",
-  "AssetEntity"."createdAt" AS "AssetEntity_createdAt",
-  "AssetEntity"."updatedAt" AS "AssetEntity_updatedAt",
-  "AssetEntity"."deletedAt" AS "AssetEntity_deletedAt",
-  "AssetEntity"."fileCreatedAt" AS "AssetEntity_fileCreatedAt",
-  "AssetEntity"."localDateTime" AS "AssetEntity_localDateTime",
-  "AssetEntity"."fileModifiedAt" AS "AssetEntity_fileModifiedAt",
-  "AssetEntity"."isFavorite" AS "AssetEntity_isFavorite",
-  "AssetEntity"."isArchived" AS "AssetEntity_isArchived",
-  "AssetEntity"."isExternal" AS "AssetEntity_isExternal",
-  "AssetEntity"."isReadOnly" AS "AssetEntity_isReadOnly",
-  "AssetEntity"."isOffline" AS "AssetEntity_isOffline",
-  "AssetEntity"."checksum" AS "AssetEntity_checksum",
-  "AssetEntity"."duration" AS "AssetEntity_duration",
-  "AssetEntity"."isVisible" AS "AssetEntity_isVisible",
-  "AssetEntity"."livePhotoVideoId" AS "AssetEntity_livePhotoVideoId",
-  "AssetEntity"."originalFileName" AS "AssetEntity_originalFileName",
-  "AssetEntity"."sidecarPath" AS "AssetEntity_sidecarPath",
-  "AssetEntity"."stackId" AS "AssetEntity_stackId"
-FROM
-  "assets" "AssetEntity"
-  LEFT JOIN "libraries" "AssetEntity__AssetEntity_library" ON "AssetEntity__AssetEntity_library"."id" = "AssetEntity"."libraryId"
-  AND (
-    "AssetEntity__AssetEntity_library"."deletedAt" IS NULL
-  )
-WHERE
-  (
-    (
-      (
-        (("AssetEntity__AssetEntity_library"."id" IN ($1)))
-      )
-    )
-  )
-  AND ("AssetEntity"."deletedAt" IS NULL)
-
 -- AssetRepository.getByLibraryIdAndOriginalPath
 SELECT DISTINCT
   "distinctAlias"."AssetEntity_id" AS "ids_AssetEntity_id"