cjones/immich
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: postgres reverse geocoding (#5301)

Browse Source 
* feat: add system metadata repository for storing key values for internal usage

* feat: add database entities for geodata

* feat: move reverse geocoding from local-reverse-geocoder to postgresql

* infra: disable synchronization for geodata_places table until typeorm supports earth column

* feat: remove cities override config as we will default all instances to cities500 now

* test: e2e tests don't clear geodata tables on reset
This commit is contained in:
Zack Pollard
2023-11-25 18:53:30 +00:00
committed by GitHub
parent 0108211c0f
commit 698226634e
46 changed files with 368 additions and 645 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/src/domain/metadata/metadata.service.spec.ts
+3 -38
View File
@@ -1,4 +1,4 @@
import { AssetType, CitiesFile, ExifEntity, SystemConfigKey } from '@app/infra/entities';
import { AssetType, ExifEntity, SystemConfigKey } from '@app/infra/entities';
import {
assetStub,
newAlbumRepositoryMock,
@@ -15,7 +15,7 @@ import { randomBytes } from 'crypto';
import { Stats } from 'fs';
import { constants } from 'fs/promises';
import { when } from 'jest-when';
import { JobName, QueueName } from '../job';
import { JobName } from '../job';
import {
IAlbumRepository,
IAssetRepository,
@@ -78,10 +78,7 @@ describe(MetadataService.name, () => {
describe('init', () => {
beforeEach(async () => {
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.REVERSE_GEOCODING_ENABLED, value: true },
{ key: SystemConfigKey.REVERSE_GEOCODING_CITIES_FILE_OVERRIDE, value: CitiesFile.CITIES_500 },
]);
configMock.load.mockResolvedValue([{ key: SystemConfigKey.REVERSE_GEOCODING_ENABLED, value: true }]);
await sut.init();
});
@@ -90,42 +87,10 @@ describe(MetadataService.name, () => {
configMock.load.mockResolvedValue([{ key: SystemConfigKey.REVERSE_GEOCODING_ENABLED, value: false }]);
await sut.init();
expect(metadataMock.deleteCache).not.toHaveBeenCalled();
expect(jobMock.pause).toHaveBeenCalledTimes(1);
expect(metadataMock.init).toHaveBeenCalledTimes(1);
expect(jobMock.resume).toHaveBeenCalledTimes(1);
});
it('should return if deleteCache is false and the cities precision has not changed', async () => {
await sut.init();
expect(metadataMock.deleteCache).not.toHaveBeenCalled();
expect(jobMock.pause).toHaveBeenCalledTimes(1);
expect(metadataMock.init).toHaveBeenCalledTimes(1);
expect(jobMock.resume).toHaveBeenCalledTimes(1);
});
it('should re-init if deleteCache is false but the cities precision has changed', async () => {
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.REVERSE_GEOCODING_CITIES_FILE_OVERRIDE, value: CitiesFile.CITIES_1000 },
]);
await sut.init();
expect(metadataMock.deleteCache).not.toHaveBeenCalled();
expect(jobMock.pause).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION);
expect(metadataMock.init).toHaveBeenCalledWith({ citiesFileOverride: CitiesFile.CITIES_1000 });
expect(jobMock.resume).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION);
});
it('should re-init and delete cache if deleteCache is true', async () => {
await sut.init(true);
expect(metadataMock.deleteCache).toHaveBeenCalled();
expect(jobMock.pause).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION);
expect(metadataMock.init).toHaveBeenCalledWith({ citiesFileOverride: CitiesFile.CITIES_500 });
expect(jobMock.resume).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION);
});
});
describe('handleLivePhotoLinking', () => {
server/src/domain/metadata/metadata.service.ts
+8 -14
View File
@@ -97,31 +97,24 @@ export class MetadataService {
this.storageCore = StorageCore.create(assetRepository, moveRepository, personRepository, storageRepository);
}
async init(deleteCache = false) {
async init() {
if (!this.subscription) {
this.subscription = this.configCore.config$.subscribe(() => this.init());
}
const { reverseGeocoding } = await this.configCore.getConfig();
const { citiesFileOverride } = reverseGeocoding;
const { enabled } = reverseGeocoding;
if (!reverseGeocoding.enabled) {
if (!enabled) {
return;
}
try {
if (deleteCache) {
await this.repository.deleteCache();
} else if (this.oldCities && this.oldCities === citiesFileOverride) {
return;
}
await this.jobRepository.pause(QueueName.METADATA_EXTRACTION);
await this.repository.init({ citiesFileOverride });
await this.repository.init();
await this.jobRepository.resume(QueueName.METADATA_EXTRACTION);
this.logger.log(`Initialized local reverse geocoder with ${citiesFileOverride}`);
this.oldCities = citiesFileOverride;
this.logger.log(`Initialized local reverse geocoder`);
} catch (error: Error | any) {
this.logger.error(`Unable to initialize reverse geocoding: ${error}`, error?.stack);
}
@@ -258,8 +251,9 @@ export class MetadataService {
}
try {
const { city, state, country } = await this.repository.reverseGeocode({ latitude, longitude });
Object.assign(exifData, { city, state, country });
const reverseGeocode = await this.repository.reverseGeocode({ latitude, longitude });
if (!reverseGeocode) return;
Object.assign(exifData, reverseGeocode);
} catch (error: Error | any) {
this.logger.warn(
`Unable to run reverse geocoding due to ${error} for asset ${asset.id} at ${asset.originalPath}`,
server/src/domain/repositories/index.ts
+1
View File
@@ -20,6 +20,7 @@ export * from './shared-link.repository';
export * from './smart-info.repository';
export * from './storage.repository';
export * from './system-config.repository';
export * from './system-metadata.repository';
export * from './tag.repository';
export * from './user-token.repository';
export * from './user.repository';
server/src/domain/repositories/metadata.repository.ts
+2 -4
View File
@@ -1,5 +1,4 @@
import { Tags } from 'exiftool-vendored';
import { InitOptions } from 'local-reverse-geocoder';
export const IMetadataRepository = 'IMetadataRepository';
@@ -31,9 +30,8 @@ export interface ImmichTags extends Omit<Tags, 'FocalLength' | 'Duration'> {
}
export interface IMetadataRepository {
init(options: Partial<InitOptions>): Promise<void>;
init(): Promise<void>;
teardown(): Promise<void>;
reverseGeocode(point: GeoPoint): Promise<ReverseGeocodeResult>;
deleteCache(): Promise<void>;
reverseGeocode(point: GeoPoint): Promise<ReverseGeocodeResult | null>;
getExifTags(path: string): Promise<ImmichTags | null>;
}
server/src/domain/repositories/system-metadata.repository.ts
+8
View File
@@ -0,0 +1,8 @@
import { SystemMetadata } from '@app/infra/entities';
export const ISystemMetadataRepository = 'ISystemMetadataRepository';
export interface ISystemMetadataRepository {
get<T extends keyof SystemMetadata>(key: T): Promise<SystemMetadata[T] | null>;
set<T extends keyof SystemMetadata>(key: T, value: SystemMetadata[T]): Promise<void>;
}
server/src/domain/system-config/dto/system-config-reverse-geocoding.dto.ts
+1 -7
View File
@@ -1,12 +1,6 @@
import { CitiesFile } from '@app/infra/entities';
import { ApiProperty } from '@nestjs/swagger';
import { IsBoolean, IsEnum } from 'class-validator';
import { IsBoolean } from 'class-validator';
export class SystemConfigReverseGeocodingDto {
@IsBoolean()
enabled!: boolean;
@IsEnum(CitiesFile)
@ApiProperty({ enum: CitiesFile, enumName: 'CitiesFile' })
citiesFileOverride!: CitiesFile;
}
server/src/domain/system-config/system-config.core.ts
-2
View File
@@ -1,6 +1,5 @@
import {
AudioCodec,
CitiesFile,
Colorspace,
CQMode,
SystemConfig,
@@ -85,7 +84,6 @@ export const defaults = Object.freeze<SystemConfig>({
},
reverseGeocoding: {
enabled: true,
citiesFileOverride: CitiesFile.CITIES_500,
},
oauth: {
enabled: false,
server/src/domain/system-config/system-config.service.spec.ts
-2
View File
@@ -1,6 +1,5 @@
import {
AudioCodec,
CitiesFile,
Colorspace,
CQMode,
SystemConfig,
@@ -85,7 +84,6 @@ const updatedConfig = Object.freeze<SystemConfig>({
},
reverseGeocoding: {
enabled: true,
citiesFileOverride: CitiesFile.CITIES_500,
},
oauth: {
autoLaunch: true,
server/src/domain/system-config/system-config.service.ts
+1 -1
View File
@@ -79,7 +79,7 @@ export class SystemConfigService {
return this.repository.fetchStyle(styleUrl);
}
return JSON.parse(await this.repository.readFile(`./assets/style-${theme}.json`));
return JSON.parse(await this.repository.readFile(`./resources/style-${theme}.json`));
}
async getCustomCss(): Promise<string> {