fix(server): don't crash when refreshing large libraries (#7934)

* add job to check for offline files

* fix lint

* only check for offline when using checkForOffline

* improve tests

* remove old test

* wip

* remove trie

* refactor batches

* also check offline status

* fix spelling

* don't do offline scan

* rename scan to check

* fix job statuses

* fix lint

* cleanup

* add test

* open-api

* fix test

* fix spinner

* reset text

* don't double batch

* fix comments from mert

* remove tries

* fix tests

* fix e2e

* fix test

* fix test

* add tests

* fix lint

* fix e2e

* interweave scans

* fix errors

* fix messages

* fix test

* add mock

* fix sql

* fix e2e

* use library batch size

* save -> update

* add file extensions

* update specs

* test for import paths

* check import paths when testing offline

* fix lint

* normalize import path

* remove console logs

* decrease batch size to 1000

* add test for import path

* add test for already-online assets

* fix merge

* fix lint

* add library job back

* add offline job to correct queue

* library spec compiles now

* move one test to new e2e

* fix comments

* fix comments

* fix lint

* refactor path validation

* fix loop bug

* remove logging

* expect responses

* fix asset mock

* take the straightforward approach

* use generator correctly

* fix vitest on file edit

* bump vitest to 1.6.0

* test for offline check

* add e2e tests for offlining assets depending on import path

* cleanup e2e test after finish

* cleanup library service

* paginate the walk generator

* fix tests

* fix typo

* refactoring handleOfflineCheck

* better testing of handleOfflineCheck

* fix lint

* handle large library deletions

* dont check if library is deleted

* fix mock

* add a 100k page size to library

* fix loading animation

* better log messages

* Better logging for offline asset removal

* fix sql and tests

* fix number format

* Remove submodule

* fix format

* chore: cleanup

* chore: fix tests

---------

Co-authored-by: Alex <alex.tran1502@gmail.com>
Co-authored-by: Jason Rasmussen <jason@rasm.me>

server/package-lock.json

@@ -45,7 +45,6 @@
         "js-yaml": "^4.1.0",
         "lodash": "^4.17.21",
         "luxon": "^3.4.2",
-        "mnemonist": "^0.39.8",
         "nest-commander": "^3.11.1",
         "nestjs-cls": "^4.3.0",
         "nestjs-otel": "^6.0.0",
@@ -10434,14 +10433,6 @@
       "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==",
       "dev": true
     },
-    "node_modules/mnemonist": {
-      "version": "0.39.8",
-      "resolved": "https://registry.npmjs.org/mnemonist/-/mnemonist-0.39.8.tgz",
-      "integrity": "sha512-vyWo2K3fjrUw8YeeZ1zF0fy6Mu59RHokURlld8ymdUPjMlD9EC9ov1/YPqTgqRvUN9nTr3Gqfz29LYAmu0PHPQ==",
-      "dependencies": {
-        "obliterator": "^2.0.1"
-      }
-    },
     "node_modules/mock-fs": {
       "version": "5.2.0",
       "resolved": "https://registry.npmjs.org/mock-fs/-/mock-fs-5.2.0.tgz",
@@ -10955,11 +10946,6 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/obliterator": {
-      "version": "2.0.4",
-      "resolved": "https://registry.npmjs.org/obliterator/-/obliterator-2.0.4.tgz",
-      "integrity": "sha512-lgHwxlxV1qIg1Eap7LgIeoBWIMFibOjbrYPIPJZcI1mmGAI2m3lNYpK12Y+GBdPQ0U1hRwSord7GIaawz962qQ=="
-    },
     "node_modules/obuf": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/obuf/-/obuf-1.1.2.tgz",
@@ -22483,14 +22469,6 @@
       "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==",
       "dev": true
     },
-    "mnemonist": {
-      "version": "0.39.8",
-      "resolved": "https://registry.npmjs.org/mnemonist/-/mnemonist-0.39.8.tgz",
-      "integrity": "sha512-vyWo2K3fjrUw8YeeZ1zF0fy6Mu59RHokURlld8ymdUPjMlD9EC9ov1/YPqTgqRvUN9nTr3Gqfz29LYAmu0PHPQ==",
-      "requires": {
-        "obliterator": "^2.0.1"
-      }
-    },
     "mock-fs": {
       "version": "5.2.0",
       "resolved": "https://registry.npmjs.org/mock-fs/-/mock-fs-5.2.0.tgz",
@@ -22855,11 +22833,6 @@
       "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.3.tgz",
       "integrity": "sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g=="
     },
-    "obliterator": {
-      "version": "2.0.4",
-      "resolved": "https://registry.npmjs.org/obliterator/-/obliterator-2.0.4.tgz",
-      "integrity": "sha512-lgHwxlxV1qIg1Eap7LgIeoBWIMFibOjbrYPIPJZcI1mmGAI2m3lNYpK12Y+GBdPQ0U1hRwSord7GIaawz962qQ=="
-    },
     "obuf": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/obuf/-/obuf-1.1.2.tgz",