cjones/immich
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: postgres reverse geocoding (#5301)

Browse Source 
* feat: add system metadata repository for storing key values for internal usage

* feat: add database entities for geodata

* feat: move reverse geocoding from local-reverse-geocoder to postgresql

* infra: disable synchronization for geodata_places table until typeorm supports earth column

* feat: remove cities override config as we will default all instances to cities500 now

* test: e2e tests don't clear geodata tables on reset
This commit is contained in:
Zack Pollard
2023-11-25 18:53:30 +00:00
committed by GitHub
parent 0108211c0f
commit 698226634e
46 changed files with 368 additions and 645 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/src/domain/metadata/metadata.service.spec.ts
+3 -38
View File
@@ -1,4 +1,4 @@
import { AssetType, CitiesFile, ExifEntity, SystemConfigKey } from '@app/infra/entities';
import { AssetType, ExifEntity, SystemConfigKey } from '@app/infra/entities';
import {
assetStub,
newAlbumRepositoryMock,
@@ -15,7 +15,7 @@ import { randomBytes } from 'crypto';
import { Stats } from 'fs';
import { constants } from 'fs/promises';
import { when } from 'jest-when';
import { JobName, QueueName } from '../job';
import { JobName } from '../job';
import {
IAlbumRepository,
IAssetRepository,
@@ -78,10 +78,7 @@ describe(MetadataService.name, () => {
describe('init', () => {
beforeEach(async () => {
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.REVERSE_GEOCODING_ENABLED, value: true },
{ key: SystemConfigKey.REVERSE_GEOCODING_CITIES_FILE_OVERRIDE, value: CitiesFile.CITIES_500 },
]);
configMock.load.mockResolvedValue([{ key: SystemConfigKey.REVERSE_GEOCODING_ENABLED, value: true }]);
await sut.init();
});
@@ -90,42 +87,10 @@ describe(MetadataService.name, () => {
configMock.load.mockResolvedValue([{ key: SystemConfigKey.REVERSE_GEOCODING_ENABLED, value: false }]);
await sut.init();
expect(metadataMock.deleteCache).not.toHaveBeenCalled();
expect(jobMock.pause).toHaveBeenCalledTimes(1);
expect(metadataMock.init).toHaveBeenCalledTimes(1);
expect(jobMock.resume).toHaveBeenCalledTimes(1);
});
it('should return if deleteCache is false and the cities precision has not changed', async () => {
await sut.init();
expect(metadataMock.deleteCache).not.toHaveBeenCalled();
expect(jobMock.pause).toHaveBeenCalledTimes(1);
expect(metadataMock.init).toHaveBeenCalledTimes(1);
expect(jobMock.resume).toHaveBeenCalledTimes(1);
});
it('should re-init if deleteCache is false but the cities precision has changed', async () => {
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.REVERSE_GEOCODING_CITIES_FILE_OVERRIDE, value: CitiesFile.CITIES_1000 },
]);
await sut.init();
expect(metadataMock.deleteCache).not.toHaveBeenCalled();
expect(jobMock.pause).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION);
expect(metadataMock.init).toHaveBeenCalledWith({ citiesFileOverride: CitiesFile.CITIES_1000 });
expect(jobMock.resume).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION);
});
it('should re-init and delete cache if deleteCache is true', async () => {
await sut.init(true);
expect(metadataMock.deleteCache).toHaveBeenCalled();
expect(jobMock.pause).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION);
expect(metadataMock.init).toHaveBeenCalledWith({ citiesFileOverride: CitiesFile.CITIES_500 });
expect(jobMock.resume).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION);
});
});
describe('handleLivePhotoLinking', () => {
server/src/domain/metadata/metadata.service.ts
+8 -14
View File
@@ -97,31 +97,24 @@ export class MetadataService {
this.storageCore = StorageCore.create(assetRepository, moveRepository, personRepository, storageRepository);
}
async init(deleteCache = false) {
async init() {
if (!this.subscription) {
this.subscription = this.configCore.config$.subscribe(() => this.init());
}
const { reverseGeocoding } = await this.configCore.getConfig();
const { citiesFileOverride } = reverseGeocoding;
const { enabled } = reverseGeocoding;
if (!reverseGeocoding.enabled) {
if (!enabled) {
return;
}
try {
if (deleteCache) {
await this.repository.deleteCache();
} else if (this.oldCities && this.oldCities === citiesFileOverride) {
return;
}
await this.jobRepository.pause(QueueName.METADATA_EXTRACTION);
await this.repository.init({ citiesFileOverride });
await this.repository.init();
await this.jobRepository.resume(QueueName.METADATA_EXTRACTION);
this.logger.log(`Initialized local reverse geocoder with ${citiesFileOverride}`);
this.oldCities = citiesFileOverride;
this.logger.log(`Initialized local reverse geocoder`);
} catch (error: Error | any) {
this.logger.error(`Unable to initialize reverse geocoding: ${error}`, error?.stack);
}
@@ -258,8 +251,9 @@ export class MetadataService {
}
try {
const { city, state, country } = await this.repository.reverseGeocode({ latitude, longitude });
Object.assign(exifData, { city, state, country });
const reverseGeocode = await this.repository.reverseGeocode({ latitude, longitude });
if (!reverseGeocode) return;
Object.assign(exifData, reverseGeocode);
} catch (error: Error | any) {
this.logger.warn(
`Unable to run reverse geocoding due to ${error} for asset ${asset.id} at ${asset.originalPath}`,