cjones/immich
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(server, web): search location (#7139)

Browse Source 
* feat: search location

* fix: tests

* feat: outclick

* location search index

* update query

* fixed query

* updated sql

* update query

* Update search.dto.ts

Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>

* coalesce

* fix: tests

* feat: add alternate names

* fix: generate sql files

* single table, add alternate names to query, cleanup

* merge main

* update sql

* pr feedback

* pr feedback

* chore: fix merge

---------

Co-authored-by: mertalev <101130780+mertalev@users.noreply.github.com>
Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>
This commit is contained in:
martin
2024-02-24 01:42:37 +01:00
committed by GitHub
parent 719dbcc4d0
commit a2934b8830
31 changed files with 1040 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/src/infra/repositories/metadata.repository.ts
+48 -61
View File
@@ -2,7 +2,7 @@ import {
citiesFile,
geodataAdmin1Path,
geodataAdmin2Path,
geodataCitites500Path,
geodataCities500Path,
geodataDatePath,
GeoPoint,
IMetadataRepository,
@@ -10,13 +10,7 @@ import {
ISystemMetadataRepository,
ReverseGeocodeResult,
} from '@app/domain';
import {
ExifEntity,
GeodataAdmin1Entity,
GeodataAdmin2Entity,
GeodataPlacesEntity,
SystemMetadataKey,
} from '@app/infra/entities';
import { ExifEntity, GeodataPlacesEntity, SystemMetadataKey } from '@app/infra/entities';
import { ImmichLogger } from '@app/infra/logger';
import { Inject } from '@nestjs/common';
import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';
@@ -26,19 +20,16 @@ import { getName } from 'i18n-iso-countries';
import { createReadStream, existsSync } from 'node:fs';
import { readFile } from 'node:fs/promises';
import * as readLine from 'node:readline';
import { DataSource, DeepPartial, QueryRunner, Repository } from 'typeorm';
import { DataSource, QueryRunner, Repository } from 'typeorm';
import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity.js';
import { DummyValue, GenerateSql } from '../infra.util';
type GeoEntity = GeodataPlacesEntity | GeodataAdmin1Entity | GeodataAdmin2Entity;
type GeoEntityClass = typeof GeodataPlacesEntity | typeof GeodataAdmin1Entity | typeof GeodataAdmin2Entity;
export class MetadataRepository implements IMetadataRepository {
constructor(
@InjectRepository(ExifEntity) private exifRepository: Repository<ExifEntity>,
@InjectRepository(GeodataPlacesEntity) private readonly geodataPlacesRepository: Repository<GeodataPlacesEntity>,
@InjectRepository(GeodataAdmin1Entity) private readonly geodataAdmin1Repository: Repository<GeodataAdmin1Entity>,
@InjectRepository(GeodataAdmin2Entity) private readonly geodataAdmin2Repository: Repository<GeodataAdmin2Entity>,
@Inject(ISystemMetadataRepository) private readonly systemMetadataRepository: ISystemMetadataRepository,
@Inject(ISystemMetadataRepository)
private readonly systemMetadataRepository: ISystemMetadataRepository,
@InjectDataSource() private dataSource: DataSource,
) {}
@@ -54,7 +45,6 @@ export class MetadataRepository implements IMetadataRepository {
return;
}
this.logger.log('Importing geodata to database from file');
await this.importGeodata();
await this.systemMetadataRepository.set(SystemMetadataKey.REVERSE_GEOCODING_STATE, {
@@ -69,12 +59,14 @@ export class MetadataRepository implements IMetadataRepository {
const queryRunner = this.dataSource.createQueryRunner();
await queryRunner.connect();
const admin1 = await this.loadAdmin(geodataAdmin1Path);
const admin2 = await this.loadAdmin(geodataAdmin2Path);
try {
await queryRunner.startTransaction();
await this.loadCities500(queryRunner);
await this.loadAdmin1(queryRunner);
await this.loadAdmin2(queryRunner);
await queryRunner.manager.clear(GeodataPlacesEntity);
await this.loadCities500(queryRunner, admin1, admin2);
await queryRunner.commitTransaction();
} catch (error) {
@@ -86,76 +78,73 @@ export class MetadataRepository implements IMetadataRepository {
}
}
private async loadGeodataToTableFromFile<T extends GeoEntity>(
private async loadGeodataToTableFromFile(
queryRunner: QueryRunner,
lineToEntityMapper: (lineSplit: string[]) => T,
lineToEntityMapper: (lineSplit: string[]) => GeodataPlacesEntity,
filePath: string,
entity: GeoEntityClass,
) {
if (!existsSync(filePath)) {
this.logger.error(`Geodata file ${filePath} not found`);
throw new Error(`Geodata file ${filePath} not found`);
}
await queryRunner.manager.clear(entity);
const input = createReadStream(filePath);
let buffer: DeepPartial<T>[] = [];
const lineReader = readLine.createInterface({ input: input });
let bufferGeodata: QueryDeepPartialEntity<GeodataPlacesEntity>[] = [];
const lineReader = readLine.createInterface({ input });
for await (const line of lineReader) {
const lineSplit = line.split('\t');
buffer.push(lineToEntityMapper(lineSplit));
if (buffer.length > 1000) {
await queryRunner.manager.save(buffer);
buffer = [];
const geoData = lineToEntityMapper(lineSplit);
bufferGeodata.push(geoData);
if (bufferGeodata.length > 1000) {
await queryRunner.manager.upsert(GeodataPlacesEntity, bufferGeodata, ['id']);
bufferGeodata = [];
}
}
await queryRunner.manager.save(buffer);
await queryRunner.manager.upsert(GeodataPlacesEntity, bufferGeodata, ['id']);
}
private async loadCities500(queryRunner: QueryRunner) {
await this.loadGeodataToTableFromFile<GeodataPlacesEntity>(
private async loadCities500(
queryRunner: QueryRunner,
admin1Map: Map<string, string>,
admin2Map: Map<string, string>,
) {
await this.loadGeodataToTableFromFile(
queryRunner,
(lineSplit: string[]) =>
this.geodataPlacesRepository.create({
id: Number.parseInt(lineSplit[0]),
name: lineSplit[1],
alternateNames: lineSplit[3],
latitude: Number.parseFloat(lineSplit[4]),
longitude: Number.parseFloat(lineSplit[5]),
countryCode: lineSplit[8],
admin1Code: lineSplit[10],
admin2Code: lineSplit[11],
modificationDate: lineSplit[18],
admin1Name: admin1Map.get(`${lineSplit[8]}.${lineSplit[10]}`),
admin2Name: admin2Map.get(`${lineSplit[8]}.${lineSplit[10]}.${lineSplit[11]}`),
}),
geodataCitites500Path,
GeodataPlacesEntity,
geodataCities500Path,
);
}
private async loadAdmin1(queryRunner: QueryRunner) {
await this.loadGeodataToTableFromFile<GeodataAdmin1Entity>(
queryRunner,
(lineSplit: string[]) =>
this.geodataAdmin1Repository.create({
key: lineSplit[0],
name: lineSplit[1],
}),
geodataAdmin1Path,
GeodataAdmin1Entity,
);
}
private async loadAdmin(filePath: string) {
if (!existsSync(filePath)) {
this.logger.error(`Geodata file ${filePath} not found`);
throw new Error(`Geodata file ${filePath} not found`);
}
private async loadAdmin2(queryRunner: QueryRunner) {
await this.loadGeodataToTableFromFile<GeodataAdmin2Entity>(
queryRunner,
(lineSplit: string[]) =>
this.geodataAdmin2Repository.create({
key: lineSplit[0],
name: lineSplit[1],
}),
geodataAdmin2Path,
GeodataAdmin2Entity,
);
const input = createReadStream(filePath);
const lineReader = readLine.createInterface({ input: input });
const adminMap = new Map<string, string>();
for await (const line of lineReader) {
const lineSplit = line.split('\t');
adminMap.set(lineSplit[0], lineSplit[1]);
}
return adminMap;
}
async teardown() {
@@ -167,8 +156,6 @@ export class MetadataRepository implements IMetadataRepository {
const response = await this.geodataPlacesRepository
.createQueryBuilder('geoplaces')
.leftJoinAndSelect('geoplaces.admin1', 'admin1')
.leftJoinAndSelect('geoplaces.admin2', 'admin2')
.where('earth_box(ll_to_earth(:latitude, :longitude), 25000) @> "earthCoord"', point)
.orderBy('earth_distance(ll_to_earth(:latitude, :longitude), "earthCoord")')
.limit(1)
@@ -183,9 +170,9 @@ export class MetadataRepository implements IMetadataRepository {
this.logger.verbose(`Raw: ${JSON.stringify(response, null, 2)}`);
const { countryCode, name: city, admin1, admin2 } = response;
const { countryCode, name: city, admin1Name, admin2Name } = response;
const country = getName(countryCode, 'en') ?? null;
const stateParts = [admin2?.name, admin1?.name].filter((name) => !!name);
const stateParts = [admin2Name, admin1Name].filter((name) => !!name);
const state = stateParts.length > 0 ? stateParts.join(', ') : null;
return { country, state, city };
server/src/infra/repositories/search.repository.ts
+29 -1
View File
@@ -12,7 +12,13 @@ import {
SmartSearchOptions,
} from '@app/domain';
import { getCLIPModelInfo } from '@app/domain/smart-info/smart-info.constant';
import { AssetEntity, AssetFaceEntity, SmartInfoEntity, SmartSearchEntity } from '@app/infra/entities';
import {
AssetEntity,
AssetFaceEntity,
GeodataPlacesEntity,
SmartInfoEntity,
SmartSearchEntity,
} from '@app/infra/entities';
import { ImmichLogger } from '@app/infra/logger';
import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
@@ -31,6 +37,7 @@ export class SearchRepository implements ISearchRepository {
@InjectRepository(AssetEntity) private assetRepository: Repository<AssetEntity>,
@InjectRepository(AssetFaceEntity) private assetFaceRepository: Repository<AssetFaceEntity>,
@InjectRepository(SmartSearchEntity) private smartSearchRepository: Repository<SmartSearchEntity>,
@InjectRepository(GeodataPlacesEntity) private readonly geodataPlacesRepository: Repository<GeodataPlacesEntity>,
) {
this.faceColumns = this.assetFaceRepository.manager.connection
.getMetadata(AssetFaceEntity)
@@ -172,6 +179,27 @@ export class SearchRepository implements ISearchRepository {
}));
}
@GenerateSql({ params: [DummyValue.STRING] })
async searchPlaces(placeName: string): Promise<GeodataPlacesEntity[]> {
return await this.geodataPlacesRepository
.createQueryBuilder('geoplaces')
.where(`f_unaccent(name) %>> f_unaccent(:placeName)`)
.orWhere(`f_unaccent("admin2Name") %>> f_unaccent(:placeName)`)
.orWhere(`f_unaccent("admin1Name") %>> f_unaccent(:placeName)`)
.orWhere(`f_unaccent("alternateNames") %>> f_unaccent(:placeName)`)
.orderBy(
`
COALESCE(f_unaccent(name) <->>> f_unaccent(:placeName), 0) +
COALESCE(f_unaccent("admin2Name") <->>> f_unaccent(:placeName), 0) +
COALESCE(f_unaccent("admin1Name") <->>> f_unaccent(:placeName), 0) +
COALESCE(f_unaccent("alternateNames") <->>> f_unaccent(:placeName), 0)
`,
)
.setParameters({ placeName })
.limit(20)
.getMany();
}
async upsert(smartInfo: Partial<SmartInfoEntity>, embedding?: Embedding): Promise<void> {
await this.repository.upsert(smartInfo, { conflictPaths: ['assetId'] });
if (!smartInfo.assetId || !embedding) {