feat: storage template file move hardening (#5917)

* fix: pgvecto.rs extension breaks typeorm schema:drop command

* fix: parse postgres bigints to javascript number types when selecting data

* feat: verify file size is the same as original asset after copying file for storage template job

* feat: allow disabling of storage template job, defaults to disabled for new instances

* fix: don't allow setting concurrency for storage template migration, can cause race conditions above 1

* feat: add checksum verification when file is copied for storage template job

* fix: extract metadata for assets that aren't visible on timeline
This commit is contained in:
Zack Pollard
2023-12-29 18:41:33 +00:00
committed by GitHub
parent 5f6bd4ae7e
commit 2e38fa73bf
36 changed files with 686 additions and 225 deletions

View File

@@ -41,7 +41,6 @@
"joi": "^17.10.0",
"lodash": "^4.17.21",
"luxon": "^3.4.2",
"mv": "^2.1.1",
"nest-commander": "^3.11.1",
"node-addon-api": "^7.0.0",
"openid-client": "^5.4.3",
@@ -72,7 +71,6 @@
"@types/lodash": "^4.14.197",
"@types/mock-fs": "^4.13.1",
"@types/multer": "^1.4.7",
"@types/mv": "^2.1.2",
"@types/node": "^20.5.7",
"@types/sharp": "^0.31.1",
"@types/supertest": "^2.0.12",
@@ -3548,12 +3546,6 @@
"@types/express": "*"
}
},
"node_modules/@types/mv": {
"version": "2.1.4",
"resolved": "https://registry.npmjs.org/@types/mv/-/mv-2.1.4.tgz",
"integrity": "sha512-MgEHBpXnQo44Q43j8G0Bvp/Yi8LYbC8hxKrRFMgDEDZMmzDKZLgiyMWtW49B37ko+QupgZ3G5rtPUnOGe5ixLw==",
"dev": true
},
"node_modules/@types/node": {
"version": "20.10.3",
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.10.3.tgz",
@@ -9293,45 +9285,6 @@
"resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz",
"integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA=="
},
"node_modules/mv": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/mv/-/mv-2.1.1.tgz",
"integrity": "sha512-at/ZndSy3xEGJ8i0ygALh8ru9qy7gWW1cmkaqBN29JmMlIvM//MEO9y1sk/avxuwnPcfhkejkLsuPxH81BrkSg==",
"dependencies": {
"mkdirp": "~0.5.1",
"ncp": "~2.0.0",
"rimraf": "~2.4.0"
},
"engines": {
"node": ">=0.8.0"
}
},
"node_modules/mv/node_modules/glob": {
"version": "6.0.4",
"resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz",
"integrity": "sha512-MKZeRNyYZAVVVG1oZeLaWie1uweH40m9AZwIwxyPbTSX4hHrVYSzLg0Ro5Z5R7XKkIX+Cc6oD1rqeDJnwsB8/A==",
"dependencies": {
"inflight": "^1.0.4",
"inherits": "2",
"minimatch": "2 || 3",
"once": "^1.3.0",
"path-is-absolute": "^1.0.0"
},
"engines": {
"node": "*"
}
},
"node_modules/mv/node_modules/rimraf": {
"version": "2.4.5",
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.4.5.tgz",
"integrity": "sha512-J5xnxTyqaiw06JjMftq7L9ouA448dw/E7dKghkP9WpKNuwmARNNg+Gk8/u5ryb9N/Yo2+z3MCwuqFK/+qPOPfQ==",
"dependencies": {
"glob": "^6.0.1"
},
"bin": {
"rimraf": "bin.js"
}
},
"node_modules/mz": {
"version": "2.7.0",
"resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz",
@@ -9355,14 +9308,6 @@
"integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==",
"dev": true
},
"node_modules/ncp": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz",
"integrity": "sha512-zIdGUrPRFTUELUvr3Gmc7KZ2Sw/h1PiVM0Af/oHB6zgnV1ikqSfRk+TOufi79aHYCW3NiOXmr1BP5nWbzojLaA==",
"bin": {
"ncp": "bin/ncp"
}
},
"node_modules/nearley": {
"version": "2.20.1",
"resolved": "https://registry.npmjs.org/nearley/-/nearley-2.20.1.tgz",
@@ -15519,12 +15464,6 @@
"@types/express": "*"
}
},
"@types/mv": {
"version": "2.1.4",
"resolved": "https://registry.npmjs.org/@types/mv/-/mv-2.1.4.tgz",
"integrity": "sha512-MgEHBpXnQo44Q43j8G0Bvp/Yi8LYbC8hxKrRFMgDEDZMmzDKZLgiyMWtW49B37ko+QupgZ3G5rtPUnOGe5ixLw==",
"dev": true
},
"@types/node": {
"version": "20.10.3",
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.10.3.tgz",
@@ -19842,38 +19781,6 @@
"resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz",
"integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA=="
},
"mv": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/mv/-/mv-2.1.1.tgz",
"integrity": "sha512-at/ZndSy3xEGJ8i0ygALh8ru9qy7gWW1cmkaqBN29JmMlIvM//MEO9y1sk/avxuwnPcfhkejkLsuPxH81BrkSg==",
"requires": {
"mkdirp": "~0.5.1",
"ncp": "~2.0.0",
"rimraf": "~2.4.0"
},
"dependencies": {
"glob": {
"version": "6.0.4",
"resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz",
"integrity": "sha512-MKZeRNyYZAVVVG1oZeLaWie1uweH40m9AZwIwxyPbTSX4hHrVYSzLg0Ro5Z5R7XKkIX+Cc6oD1rqeDJnwsB8/A==",
"requires": {
"inflight": "^1.0.4",
"inherits": "2",
"minimatch": "2 || 3",
"once": "^1.3.0",
"path-is-absolute": "^1.0.0"
}
},
"rimraf": {
"version": "2.4.5",
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.4.5.tgz",
"integrity": "sha512-J5xnxTyqaiw06JjMftq7L9ouA448dw/E7dKghkP9WpKNuwmARNNg+Gk8/u5ryb9N/Yo2+z3MCwuqFK/+qPOPfQ==",
"requires": {
"glob": "^6.0.1"
}
}
}
},
"mz": {
"version": "2.7.0",
"resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz",
@@ -19897,11 +19804,6 @@
"integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==",
"dev": true
},
"ncp": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz",
"integrity": "sha512-zIdGUrPRFTUELUvr3Gmc7KZ2Sw/h1PiVM0Af/oHB6zgnV1ikqSfRk+TOufi79aHYCW3NiOXmr1BP5nWbzojLaA=="
},
"nearley": {
"version": "2.20.1",
"resolved": "https://registry.npmjs.org/nearley/-/nearley-2.20.1.tgz",