feat: sync assets, partner assets, exif, and partner exif (#16658)

* feat: sync assets, partner assets, exif, and partner exif

Co-authored-by: Zack Pollard <zack@futo.org>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>

* refactor: remove duplicate where clause and orderBy statements in sync queries

* fix: asset deletes not filtering by ownerId

---------

Co-authored-by: Zack Pollard <zack@futo.org>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
Co-authored-by: Zack Pollard <zackpollard@ymail.com>

server/test/factory.ts

@@ -55,7 +55,7 @@ class CustomWritable extends Writable {
   }
 }
 
-type Asset = Insertable<Assets>;
+type Asset = Partial<Insertable<Assets>>;
 type User = Partial<Insertable<Users>>;
 type Session = Omit<Insertable<Sessions>, 'token'> & { token?: string };
 type Partner = Insertable<Partners>;
@@ -160,10 +160,6 @@ export class TestFactory {
   }
 
   async create() {
-    for (const asset of this.assets) {
-      await this.context.createAsset(asset);
-    }
-
     for (const user of this.users) {
       await this.context.createUser(user);
     }
@@ -176,6 +172,10 @@ export class TestFactory {
       await this.context.createSession(session);
     }
 
+    for (const asset of this.assets) {
+      await this.context.createAsset(asset);
+    }
+
     return this.context;
   }
 }
@@ -212,7 +212,7 @@ export class TestContext {
   versionHistory: VersionHistoryRepository;
   view: ViewRepository;
 
-  private constructor(private db: Kysely<DB>) {
+  private constructor(public db: Kysely<DB>) {
     const logger = newLoggingRepositoryMock() as unknown as LoggingRepository;
     const config = new ConfigRepository();
 

server/test/medium/specs/audit.database.spec.ts

@@ -0,0 +1,74 @@
+import { TestContext, TestFactory } from 'test/factory';
+import { getKyselyDB } from 'test/utils';
+
+describe('audit', () => {
+  let context: TestContext;
+
+  beforeAll(async () => {
+    const db = await getKyselyDB();
+    context = await TestContext.from(db).create();
+  });
+
+  describe('partners_audit', () => {
+    it('should not cascade user deletes to partners_audit', async () => {
+      const user1 = TestFactory.user();
+      const user2 = TestFactory.user();
+
+      await context
+        .getFactory()
+        .withUser(user1)
+        .withUser(user2)
+        .withPartner({ sharedById: user1.id, sharedWithId: user2.id })
+        .create();
+
+      await context.user.delete(user1, true);
+
+      await expect(
+        context.db.selectFrom('partners_audit').select(['id']).where('sharedById', '=', user1.id).execute(),
+      ).resolves.toHaveLength(0);
+    });
+  });
+
+  describe('assets_audit', () => {
+    it('should not cascade user deletes to assets_audit', async () => {
+      const user = TestFactory.user();
+      const asset = TestFactory.asset({ ownerId: user.id });
+
+      await context.getFactory().withUser(user).withAsset(asset).create();
+
+      await context.user.delete(user, true);
+
+      await expect(
+        context.db.selectFrom('assets_audit').select(['id']).where('assetId', '=', asset.id).execute(),
+      ).resolves.toHaveLength(0);
+    });
+  });
+
+  describe('exif', () => {
+    it('should automatically set updatedAt and updateId when the row is updated', async () => {
+      const user = TestFactory.user();
+      const asset = TestFactory.asset({ ownerId: user.id });
+      const exif = { assetId: asset.id, make: 'Canon' };
+
+      await context.getFactory().withUser(user).withAsset(asset).create();
+      await context.asset.upsertExif(exif);
+
+      const before = await context.db
+        .selectFrom('exif')
+        .select(['updatedAt', 'updateId'])
+        .where('assetId', '=', asset.id)
+        .executeTakeFirstOrThrow();
+
+      await context.asset.upsertExif({ assetId: asset.id, make: 'Canon 2' });
+
+      const after = await context.db
+        .selectFrom('exif')
+        .select(['updatedAt', 'updateId'])
+        .where('assetId', '=', asset.id)
+        .executeTakeFirstOrThrow();
+
+      expect(before.updateId).not.toEqual(after.updateId);
+      expect(before.updatedAt).not.toEqual(after.updatedAt);
+    });
+  });
+});

server/test/medium/specs/sync.service.spec.ts

@@ -1,6 +1,6 @@
 import { AuthDto } from 'src/dtos/auth.dto';
-import { SyncRequestType } from 'src/enum';
-import { SyncService } from 'src/services/sync.service';
+import { SyncEntityType, SyncRequestType } from 'src/enum';
+import { SYNC_TYPES_ORDER, SyncService } from 'src/services/sync.service';
 import { TestContext, TestFactory } from 'test/factory';
 import { getKyselyDB, newTestService } from 'test/utils';
 
@@ -33,7 +33,15 @@ const setup = async () => {
 };
 
 describe(SyncService.name, () => {
-  describe.concurrent('users', () => {
+  it('should have all the types in the ordering variable', () => {
+    for (const key in SyncRequestType) {
+      expect(SYNC_TYPES_ORDER).includes(key);
+    }
+
+    expect(SYNC_TYPES_ORDER.length).toBe(Object.keys(SyncRequestType).length);
+  });
+
+  describe.concurrent(SyncEntityType.UserV1, () => {
     it('should detect and sync the first user', async () => {
       const { context, auth, sut, testSync } = await setup();
 
@@ -189,7 +197,7 @@ describe(SyncService.name, () => {
     });
   });
 
-  describe.concurrent('partners', () => {
+  describe.concurrent(SyncEntityType.PartnerV1, () => {
     it('should detect and sync the first partner', async () => {
       const { auth, context, sut, testSync } = await setup();
 
@@ -349,7 +357,7 @@ describe(SyncService.name, () => {
       );
     });
 
-    it('should not sync a partner for an unrelated user', async () => {
+    it('should not sync a partner or partner delete for an unrelated user', async () => {
       const { auth, context, testSync } = await setup();
 
       const user2 = await context.createUser();
@@ -357,9 +365,436 @@ describe(SyncService.name, () => {
 
       await context.createPartner({ sharedById: user2.id, sharedWithId: user3.id });
 
-      const response = await testSync(auth, [SyncRequestType.PartnersV1]);
+      expect(await testSync(auth, [SyncRequestType.PartnersV1])).toHaveLength(0);
+
+      await context.partner.remove({ sharedById: user2.id, sharedWithId: user3.id });
+
+      expect(await testSync(auth, [SyncRequestType.PartnersV1])).toHaveLength(0);
+    });
+
+    it('should not sync a partner delete after a user is deleted', async () => {
+      const { auth, context, testSync } = await setup();
+
+      const user2 = await context.createUser();
+      await context.createPartner({ sharedById: user2.id, sharedWithId: auth.user.id });
+      await context.user.delete({ id: user2.id }, true);
+
+      expect(await testSync(auth, [SyncRequestType.PartnersV1])).toHaveLength(0);
+    });
+  });
+
+  describe.concurrent(SyncEntityType.AssetV1, () => {
+    it('should detect and sync the first asset', async () => {
+      const { auth, context, sut, testSync } = await setup();
+
+      const checksum = '1115vHcVkZzNp3Q9G+FEA0nu6zUbGb4Tj4UOXkN0wRA=';
+      const thumbhash = '2225vHcVkZzNp3Q9G+FEA0nu6zUbGb4Tj4UOXkN0wRA=';
+      const date = new Date().toISOString();
+
+      const asset = TestFactory.asset({
+        ownerId: auth.user.id,
+        checksum: Buffer.from(checksum, 'base64'),
+        thumbhash: Buffer.from(thumbhash, 'base64'),
+        fileCreatedAt: date,
+        fileModifiedAt: date,
+        deletedAt: null,
+      });
+      await context.createAsset(asset);
+
+      const initialSyncResponse = await testSync(auth, [SyncRequestType.AssetsV1]);
+
+      expect(initialSyncResponse).toHaveLength(1);
+      expect(initialSyncResponse).toEqual(
+        expect.arrayContaining([
+          {
+            ack: expect.any(String),
+            data: {
+              id: asset.id,
+              ownerId: asset.ownerId,
+              thumbhash,
+              checksum,
+              deletedAt: null,
+              fileCreatedAt: date,
+              fileModifiedAt: date,
+              isFavorite: false,
+              isVisible: true,
+              localDateTime: null,
+              type: asset.type,
+            },
+            type: 'AssetV1',
+          },
+        ]),
+      );
+
+      const acks = [initialSyncResponse[0].ack];
+      await sut.setAcks(auth, { acks });
+
+      const ackSyncResponse = await testSync(auth, [SyncRequestType.AssetsV1]);
+
+      expect(ackSyncResponse).toHaveLength(0);
+    });
+
+    it('should detect and sync a deleted asset', async () => {
+      const { auth, context, sut, testSync } = await setup();
+
+      const asset = TestFactory.asset({ ownerId: auth.user.id });
+      await context.createAsset(asset);
+      await context.asset.remove(asset);
+
+      const response = await testSync(auth, [SyncRequestType.AssetsV1]);
+
+      expect(response).toHaveLength(1);
+      expect(response).toEqual(
+        expect.arrayContaining([
+          {
+            ack: expect.any(String),
+            data: {
+              assetId: asset.id,
+            },
+            type: 'AssetDeleteV1',
+          },
+        ]),
+      );
+
+      const acks = response.map(({ ack }) => ack);
+      await sut.setAcks(auth, { acks });
+
+      const ackSyncResponse = await testSync(auth, [SyncRequestType.AssetsV1]);
+
+      expect(ackSyncResponse).toHaveLength(0);
+    });
+
+    it('should not sync an asset or asset delete for an unrelated user', async () => {
+      const { auth, context, testSync } = await setup();
+
+      const user2 = await context.createUser();
+      const session = TestFactory.session({ userId: user2.id });
+      const auth2 = TestFactory.auth({ session, user: user2 });
+
+      const asset = TestFactory.asset({ ownerId: user2.id });
+      await context.createAsset(asset);
+
+      expect(await testSync(auth2, [SyncRequestType.AssetsV1])).toHaveLength(1);
+      expect(await testSync(auth, [SyncRequestType.AssetsV1])).toHaveLength(0);
+
+      await context.asset.remove(asset);
+      expect(await testSync(auth2, [SyncRequestType.AssetsV1])).toHaveLength(1);
+      expect(await testSync(auth, [SyncRequestType.AssetsV1])).toHaveLength(0);
+    });
+  });
+
+  describe.concurrent(SyncRequestType.PartnerAssetsV1, () => {
+    it('should detect and sync the first partner asset', async () => {
+      const { auth, context, sut, testSync } = await setup();
+
+      const checksum = '1115vHcVkZzNp3Q9G+FEA0nu6zUbGb4Tj4UOXkN0wRA=';
+      const thumbhash = '2225vHcVkZzNp3Q9G+FEA0nu6zUbGb4Tj4UOXkN0wRA=';
+      const date = new Date().toISOString();
+
+      const user2 = await context.createUser();
+
+      const asset = TestFactory.asset({
+        ownerId: user2.id,
+        checksum: Buffer.from(checksum, 'base64'),
+        thumbhash: Buffer.from(thumbhash, 'base64'),
+        fileCreatedAt: date,
+        fileModifiedAt: date,
+        deletedAt: null,
+      });
+      await context.createAsset(asset);
+      await context.partner.create({ sharedById: user2.id, sharedWithId: auth.user.id });
+
+      const initialSyncResponse = await testSync(auth, [SyncRequestType.PartnerAssetsV1]);
+
+      expect(initialSyncResponse).toHaveLength(1);
+      expect(initialSyncResponse).toEqual(
+        expect.arrayContaining([
+          {
+            ack: expect.any(String),
+            data: {
+              id: asset.id,
+              ownerId: asset.ownerId,
+              thumbhash,
+              checksum,
+              deletedAt: null,
+              fileCreatedAt: date,
+              fileModifiedAt: date,
+              isFavorite: false,
+              isVisible: true,
+              localDateTime: null,
+              type: asset.type,
+            },
+            type: SyncEntityType.PartnerAssetV1,
+          },
+        ]),
+      );
+
+      const acks = [initialSyncResponse[0].ack];
+      await sut.setAcks(auth, { acks });
+
+      const ackSyncResponse = await testSync(auth, [SyncRequestType.PartnerAssetsV1]);
+
+      expect(ackSyncResponse).toHaveLength(0);
+    });
+
+    it('should detect and sync a deleted partner asset', async () => {
+      const { auth, context, sut, testSync } = await setup();
+
+      const user2 = await context.createUser();
+      const asset = TestFactory.asset({ ownerId: user2.id });
+      await context.createAsset(asset);
+      await context.partner.create({ sharedById: user2.id, sharedWithId: auth.user.id });
+      await context.asset.remove(asset);
+
+      const response = await testSync(auth, [SyncRequestType.PartnerAssetsV1]);
+
+      expect(response).toHaveLength(1);
+      expect(response).toEqual(
+        expect.arrayContaining([
+          {
+            ack: expect.any(String),
+            data: {
+              assetId: asset.id,
+            },
+            type: SyncEntityType.PartnerAssetDeleteV1,
+          },
+        ]),
+      );
+
+      const acks = response.map(({ ack }) => ack);
+      await sut.setAcks(auth, { acks });
+
+      const ackSyncResponse = await testSync(auth, [SyncRequestType.PartnerAssetsV1]);
+
+      expect(ackSyncResponse).toHaveLength(0);
+    });
+
+    it('should not sync a deleted partner asset due to a user delete', async () => {
+      const { auth, context, testSync } = await setup();
+
+      const user2 = await context.createUser();
+      await context.partner.create({ sharedById: user2.id, sharedWithId: auth.user.id });
+      await context.createAsset({ ownerId: user2.id });
+      await context.user.delete({ id: user2.id }, true);
+
+      const response = await testSync(auth, [SyncRequestType.PartnerAssetsV1]);
 
       expect(response).toHaveLength(0);
     });
+
+    it('should not sync a deleted partner asset due to a partner delete (unshare)', async () => {
+      const { auth, context, testSync } = await setup();
+
+      const user2 = await context.createUser();
+      await context.createAsset({ ownerId: user2.id });
+      const partner = { sharedById: user2.id, sharedWithId: auth.user.id };
+      await context.partner.create(partner);
+
+      await expect(testSync(auth, [SyncRequestType.PartnerAssetsV1])).resolves.toHaveLength(1);
+
+      await context.partner.remove(partner);
+
+      await expect(testSync(auth, [SyncRequestType.PartnerAssetsV1])).resolves.toHaveLength(0);
+    });
+
+    it('should not sync an asset or asset delete for own user', async () => {
+      const { auth, context, testSync } = await setup();
+
+      const user2 = await context.createUser();
+      const asset = await context.createAsset({ ownerId: auth.user.id });
+      const partner = { sharedById: user2.id, sharedWithId: auth.user.id };
+      await context.partner.create(partner);
+
+      await expect(testSync(auth, [SyncRequestType.AssetsV1])).resolves.toHaveLength(1);
+      await expect(testSync(auth, [SyncRequestType.PartnerAssetsV1])).resolves.toHaveLength(0);
+
+      await context.asset.remove(asset);
+
+      await expect(testSync(auth, [SyncRequestType.AssetsV1])).resolves.toHaveLength(1);
+      await expect(testSync(auth, [SyncRequestType.PartnerAssetsV1])).resolves.toHaveLength(0);
+    });
+
+    it('should not sync an asset or asset delete for unrelated user', async () => {
+      const { auth, context, testSync } = await setup();
+
+      const user2 = await context.createUser();
+      const session = TestFactory.session({ userId: user2.id });
+      const auth2 = TestFactory.auth({ session, user: user2 });
+      const asset = await context.createAsset({ ownerId: user2.id });
+
+      await expect(testSync(auth2, [SyncRequestType.AssetsV1])).resolves.toHaveLength(1);
+      await expect(testSync(auth, [SyncRequestType.PartnerAssetsV1])).resolves.toHaveLength(0);
+
+      await context.asset.remove(asset);
+
+      await expect(testSync(auth2, [SyncRequestType.AssetsV1])).resolves.toHaveLength(1);
+      await expect(testSync(auth, [SyncRequestType.PartnerAssetsV1])).resolves.toHaveLength(0);
+    });
+  });
+
+  describe.concurrent(SyncRequestType.AssetExifsV1, () => {
+    it('should detect and sync the first asset exif', async () => {
+      const { auth, context, sut, testSync } = await setup();
+
+      const asset = TestFactory.asset({ ownerId: auth.user.id });
+      const exif = { assetId: asset.id, make: 'Canon' };
+
+      await context.createAsset(asset);
+      await context.asset.upsertExif(exif);
+
+      const initialSyncResponse = await testSync(auth, [SyncRequestType.AssetExifsV1]);
+
+      expect(initialSyncResponse).toHaveLength(1);
+      expect(initialSyncResponse).toEqual(
+        expect.arrayContaining([
+          {
+            ack: expect.any(String),
+            data: {
+              assetId: asset.id,
+              city: null,
+              country: null,
+              dateTimeOriginal: null,
+              description: '',
+              exifImageHeight: null,
+              exifImageWidth: null,
+              exposureTime: null,
+              fNumber: null,
+              fileSizeInByte: null,
+              focalLength: null,
+              fps: null,
+              iso: null,
+              latitude: null,
+              lensModel: null,
+              longitude: null,
+              make: 'Canon',
+              model: null,
+              modifyDate: null,
+              orientation: null,
+              profileDescription: null,
+              projectionType: null,
+              rating: null,
+              state: null,
+              timeZone: null,
+            },
+            type: SyncEntityType.AssetExifV1,
+          },
+        ]),
+      );
+
+      const acks = [initialSyncResponse[0].ack];
+      await sut.setAcks(auth, { acks });
+
+      const ackSyncResponse = await testSync(auth, [SyncRequestType.AssetExifsV1]);
+
+      expect(ackSyncResponse).toHaveLength(0);
+    });
+
+    it('should only sync asset exif for own user', async () => {
+      const { auth, context, testSync } = await setup();
+
+      const user2 = await context.createUser();
+      const session = TestFactory.session({ userId: user2.id });
+      const auth2 = TestFactory.auth({ session, user: user2 });
+
+      await context.partner.create({ sharedById: user2.id, sharedWithId: auth.user.id });
+      const asset = TestFactory.asset({ ownerId: user2.id });
+      const exif = { assetId: asset.id, make: 'Canon' };
+
+      await context.createAsset(asset);
+      await context.asset.upsertExif(exif);
+
+      await expect(testSync(auth2, [SyncRequestType.AssetExifsV1])).resolves.toHaveLength(1);
+      await expect(testSync(auth, [SyncRequestType.AssetExifsV1])).resolves.toHaveLength(0);
+    });
+  });
+
+  describe.concurrent(SyncRequestType.PartnerAssetExifsV1, () => {
+    it('should detect and sync the first partner asset exif', async () => {
+      const { auth, context, sut, testSync } = await setup();
+
+      const user2 = await context.createUser();
+      await context.partner.create({ sharedById: user2.id, sharedWithId: auth.user.id });
+      const asset = TestFactory.asset({ ownerId: user2.id });
+      await context.createAsset(asset);
+      const exif = { assetId: asset.id, make: 'Canon' };
+      await context.asset.upsertExif(exif);
+
+      const initialSyncResponse = await testSync(auth, [SyncRequestType.PartnerAssetExifsV1]);
+
+      expect(initialSyncResponse).toHaveLength(1);
+      expect(initialSyncResponse).toEqual(
+        expect.arrayContaining([
+          {
+            ack: expect.any(String),
+            data: {
+              assetId: asset.id,
+              city: null,
+              country: null,
+              dateTimeOriginal: null,
+              description: '',
+              exifImageHeight: null,
+              exifImageWidth: null,
+              exposureTime: null,
+              fNumber: null,
+              fileSizeInByte: null,
+              focalLength: null,
+              fps: null,
+              iso: null,
+              latitude: null,
+              lensModel: null,
+              longitude: null,
+              make: 'Canon',
+              model: null,
+              modifyDate: null,
+              orientation: null,
+              profileDescription: null,
+              projectionType: null,
+              rating: null,
+              state: null,
+              timeZone: null,
+            },
+            type: SyncEntityType.PartnerAssetExifV1,
+          },
+        ]),
+      );
+
+      const acks = [initialSyncResponse[0].ack];
+      await sut.setAcks(auth, { acks });
+
+      const ackSyncResponse = await testSync(auth, [SyncRequestType.PartnerAssetExifsV1]);
+
+      expect(ackSyncResponse).toHaveLength(0);
+    });
+
+    it('should not sync partner asset exif for own user', async () => {
+      const { auth, context, testSync } = await setup();
+
+      const user2 = await context.createUser();
+      await context.partner.create({ sharedById: user2.id, sharedWithId: auth.user.id });
+      const asset = TestFactory.asset({ ownerId: auth.user.id });
+      const exif = { assetId: asset.id, make: 'Canon' };
+      await context.createAsset(asset);
+      await context.asset.upsertExif(exif);
+
+      await expect(testSync(auth, [SyncRequestType.AssetExifsV1])).resolves.toHaveLength(1);
+      await expect(testSync(auth, [SyncRequestType.PartnerAssetExifsV1])).resolves.toHaveLength(0);
+    });
+
+    it('should not sync partner asset exif for unrelated user', async () => {
+      const { auth, context, testSync } = await setup();
+
+      const user2 = await context.createUser();
+      const user3 = await context.createUser();
+      const session = TestFactory.session({ userId: user3.id });
+      const authUser3 = TestFactory.auth({ session, user: user3 });
+      await context.partner.create({ sharedById: user2.id, sharedWithId: auth.user.id });
+      const asset = TestFactory.asset({ ownerId: user3.id });
+      const exif = { assetId: asset.id, make: 'Canon' };
+      await context.createAsset(asset);
+      await context.asset.upsertExif(exif);
+
+      await expect(testSync(authUser3, [SyncRequestType.AssetExifsV1])).resolves.toHaveLength(1);
+      await expect(testSync(auth, [SyncRequestType.PartnerAssetExifsV1])).resolves.toHaveLength(0);
+    });
   });
 });

server/test/repositories/sync.repository.mock.ts

@@ -11,5 +11,11 @@ export const newSyncRepositoryMock = (): Mocked<RepositoryInterface<SyncReposito
     getUserDeletes: vitest.fn(),
     getPartnerUpserts: vitest.fn(),
     getPartnerDeletes: vitest.fn(),
+    getPartnerAssetsUpserts: vitest.fn(),
+    getPartnerAssetDeletes: vitest.fn(),
+    getAssetDeletes: vitest.fn(),
+    getAssetUpserts: vitest.fn(),
+    getAssetExifsUpserts: vitest.fn(),
+    getPartnerAssetExifsUpserts: vitest.fn(),
   };
 };