Slave/db storage #3

Merged
nobswebdev merged 3 commits from slave/db-storage into master 2026-10-05 21:15:04 +00:00
11 changed files with 3906 additions and 3 deletions
Showing only changes of commit a8070da39d - Show all commits
+1 -1
View File
@@ -7,7 +7,7 @@
/backend/node_modules /backend/node_modules
/backend/dist /backend/dist
/backend/*.tsbuildinfo /backend/*.tsbuildinfo
/backend/storage /backend/uploads
/frontend/node_modules /frontend/node_modules
/frontend/dist /frontend/dist
+3649
View File
File diff suppressed because it is too large Load Diff
+9 -1
View File
@@ -12,7 +12,12 @@
"typeorm:run-migrations": "npm run typeorm migration:run -- -d ./src/database/DataSource.ts", "typeorm:run-migrations": "npm run typeorm migration:run -- -d ./src/database/DataSource.ts",
"typeorm:generate-migration": "npm run typeorm -- -d ./src/database/DataSource.ts migration:generate ./src/database/migrations/$npm_config_name", "typeorm:generate-migration": "npm run typeorm -- -d ./src/database/DataSource.ts migration:generate ./src/database/migrations/$npm_config_name",
"typeorm:revert-migration": "npm run typeorm -- -d ./src/database/DataSource.ts migration:revert", "typeorm:revert-migration": "npm run typeorm -- -d ./src/database/DataSource.ts migration:revert",
"typeorm:create-migration": "npm run typeorm -- migration:create ./src/database/migrations/$npm_config_name" "typeorm:create-migration": "npm run typeorm -- migration:create ./src/database/migrations/$npm_config_name",
"test": "jest"
},
"jest": {
"preset": "ts-jest",
"rootDir": "src"
}, },
"dependencies": { "dependencies": {
"@nestjs/common": "^11.0.1", "@nestjs/common": "^11.0.1",
@@ -30,8 +35,11 @@
"@nestjs/cli": "^11.0.24", "@nestjs/cli": "^11.0.24",
"@nestjs/schematics": "^11.0.0", "@nestjs/schematics": "^11.0.0",
"@types/express": "^5.0.0", "@types/express": "^5.0.0",
"@types/jest": "^30.0.0",
"@types/node": "^22.10.7", "@types/node": "^22.10.7",
"@types/pg": "^8.23.1", "@types/pg": "^8.23.1",
"jest": "^30.5.2",
"ts-jest": "^29.4.14",
"ts-node": "^10.9.2", "ts-node": "^10.9.2",
"tsconfig-paths": "^4.2.0", "tsconfig-paths": "^4.2.0",
"typescript": "~6.0.2" "typescript": "~6.0.2"
+2
View File
@@ -3,6 +3,7 @@ import { ConfigModule, ConfigService, registerAs } from '@nestjs/config';
import { TypeOrmModule } from '@nestjs/typeorm'; import { TypeOrmModule } from '@nestjs/typeorm';
import { getAppConfig, getPostgresConfig } from './config'; import { getAppConfig, getPostgresConfig } from './config';
import { HealthCheckModule } from './modules/healthCheck/HealthCheckModule'; import { HealthCheckModule } from './modules/healthCheck/HealthCheckModule';
import { StorageModule } from './modules/storage/StorageModule';
import { UploadModule } from './modules/upload/UploadModule'; import { UploadModule } from './modules/upload/UploadModule';
import { Config } from './types/Config'; import { Config } from './types/Config';
@@ -17,6 +18,7 @@ import { Config } from './types/Config';
inject: [ConfigService] inject: [ConfigService]
}), }),
HealthCheckModule, HealthCheckModule,
StorageModule,
UploadModule UploadModule
] ]
}) })
+61
View File
@@ -0,0 +1,61 @@
import { join, resolve } from 'node:path';
import { getUploadsDir, resolveStoragePath } from './uploadPaths';
describe('uploadPaths', () => {
describe('getUploadsDir', () => {
it('returns uploads under process cwd', () => {
expect(getUploadsDir()).toBe(join(process.cwd(), 'uploads'));
});
it('returns a stable path on repeated calls', () => {
expect(getUploadsDir()).toBe(getUploadsDir());
});
});
describe('resolveStoragePath', () => {
it('resolves a storage key under uploads dir', () => {
const storageKey = 'a1b2c3d4-e5f6-7890-abcd-ef1234567890';
expect(resolveStoragePath(storageKey)).toBe(resolve(getUploadsDir(), storageKey));
});
it('resolves a nested storage key under uploads dir', () => {
const storageKey = 'nested/a1b2c3d4-e5f6-7890-abcd-ef1234567890';
expect(resolveStoragePath(storageKey)).toBe(resolve(getUploadsDir(), storageKey));
});
it('rejects path traversal with parent segments', () => {
expect(() => resolveStoragePath('../../etc/passwd')).toThrow('Invalid storage key');
expect(() => resolveStoragePath('../outside')).toThrow('Invalid storage key');
});
it('rejects absolute storage keys outside uploads dir', () => {
expect(() => resolveStoragePath('/etc/passwd')).toThrow('Invalid storage key');
});
it('rejects empty storage key', () => {
expect(() => resolveStoragePath('')).toThrow('Invalid storage key');
});
it('rejects traversal hidden after a benign prefix', () => {
expect(() => resolveStoragePath('safe/../../etc/passwd')).toThrow('Invalid storage key');
});
it.each(['..', 'foo/../../../etc/passwd'])('rejects suspicious storage key: %s', storageKey => {
expect(() => resolveStoragePath(storageKey)).toThrow('Invalid storage key');
});
it('treats encoded traversal literals as opaque file names', () => {
const storageKey = '%2e%2e%2fetc/passwd';
expect(resolveStoragePath(storageKey)).toBe(resolve(getUploadsDir(), storageKey));
});
it('allows dot characters inside a single file name segment', () => {
const storageKey = 'my.file.name.bin';
expect(resolveStoragePath(storageKey)).toBe(resolve(getUploadsDir(), storageKey));
});
});
});
+26
View File
@@ -0,0 +1,26 @@
import { join, resolve, sep } from 'node:path';
export const getUploadsDir = (): string => join(process.cwd(), 'uploads');
const validatePathTraversal = (baseDir: string, absolutePath: string): void => {
const resolvedBaseDir = resolve(baseDir);
if (!absolutePath.startsWith(`${resolvedBaseDir}${sep}`)) {
throw new Error('Invalid storage key');
}
};
const resolvePathWithinDir = (baseDir: string, storageKey: string): string => {
const resolvedBaseDir = resolve(baseDir);
const absolutePath = resolve(resolvedBaseDir, storageKey);
validatePathTraversal(resolvedBaseDir, absolutePath);
return absolutePath;
};
export const resolveStoragePath = (storageKey: string): string => {
const uploadsDir = getUploadsDir();
return resolvePathWithinDir(uploadsDir, storageKey);
};
@@ -0,0 +1,8 @@
import { Module } from '@nestjs/common';
import { StorageService } from './services/StorageService';
@Module({
providers: [StorageService],
exports: [StorageService]
})
export class StorageModule {}
@@ -0,0 +1,120 @@
import { access, mkdtemp, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { getUploadsDir, resolveStoragePath } from '../../../config/uploadPaths';
import { StorageService } from './StorageService';
describe('StorageService', () => {
let service: StorageService;
let previousCwd: string;
let tempRoot: string;
beforeEach(async () => {
service = new StorageService();
previousCwd = process.cwd();
tempRoot = await mkdtemp(join(tmpdir(), 'storage-service-'));
process.chdir(tempRoot);
});
afterEach(async () => {
process.chdir(previousCwd);
await rm(tempRoot, { recursive: true, force: true });
});
it('writes and reads ciphertext bytes', async () => {
const storageKey = 'a1b2c3d4-e5f6-7890-abcd-ef1234567890';
const data = Buffer.from('ciphertext-bytes');
await service.write(storageKey, data);
expect(await service.read(storageKey)).toEqual(data);
});
it('creates the uploads directory on write', async () => {
await service.write('file-id', Buffer.from('x'));
await expect(access(getUploadsDir())).resolves.toBeUndefined();
});
it('writes nested storage keys under uploads', async () => {
const storageKey = 'nested/file-id';
const data = Buffer.from('nested');
await service.write(storageKey, data);
expect(await service.read(storageKey)).toEqual(data);
});
it('unlinks a stored file', async () => {
const storageKey = 'to-delete';
await service.write(storageKey, Buffer.from('delete-me'));
await service.unlink(storageKey);
await expect(access(resolveStoragePath(storageKey))).rejects.toMatchObject({ code: 'ENOENT' });
});
it.each(['../../escape', '../outside', '/etc/passwd', ''])(
'rejects invalid storage key on write: %s',
async storageKey => {
await expect(service.write(storageKey, Buffer.from('x'))).rejects.toThrow('Invalid storage key');
}
);
it.each(['../../escape', '../outside', '/etc/passwd', ''])(
'rejects invalid storage key on read: %s',
storageKey => {
expect(() => service.read(storageKey)).toThrow('Invalid storage key');
}
);
it.each(['../../escape', '../outside', '/etc/passwd', ''])(
'rejects invalid storage key on unlink: %s',
storageKey => {
expect(() => service.unlink(storageKey)).toThrow('Invalid storage key');
}
);
it('throws when reading a missing file', async () => {
await expect(service.read('missing-id')).rejects.toMatchObject({ code: 'ENOENT' });
});
it('throws when unlinking a missing file', async () => {
await expect(service.unlink('missing-id')).rejects.toMatchObject({ code: 'ENOENT' });
});
it('overwrites an existing storage key on second write', async () => {
const storageKey = 'overwrite-me';
await service.write(storageKey, Buffer.from('first'));
await service.write(storageKey, Buffer.from('second'));
expect(await service.read(storageKey)).toEqual(Buffer.from('second'));
});
it('keeps separate storage keys isolated', async () => {
await service.write('key-a', Buffer.from('a'));
await service.write('key-b', Buffer.from('b'));
expect(await service.read('key-a')).toEqual(Buffer.from('a'));
expect(await service.read('key-b')).toEqual(Buffer.from('b'));
});
it('round-trips an empty buffer', async () => {
const storageKey = 'empty';
const data = Buffer.alloc(0);
await service.write(storageKey, data);
expect(await service.read(storageKey)).toEqual(data);
});
it('round-trips binary ciphertext bytes', async () => {
const storageKey = 'binary';
const data = Buffer.from([0x00, 0xff, 0x10, 0x89, 0xab]);
await service.write(storageKey, data);
expect(await service.read(storageKey)).toEqual(data);
});
});
@@ -0,0 +1,28 @@
import { mkdir, readFile, unlink as fsUnlink, writeFile } from 'node:fs/promises';
import { dirname } from 'node:path';
import { Injectable } from '@nestjs/common';
import { resolveStoragePath } from '../../../config/uploadPaths';
@Injectable()
export class StorageService {
async write(storageKey: string, data: Buffer): Promise<void> {
const absolutePath = resolveStoragePath(storageKey);
const dir = dirname(absolutePath);
await mkdir(dir, { recursive: true });
return writeFile(absolutePath, data);
}
read(storageKey: string): Promise<Buffer> {
const absolutePath = resolveStoragePath(storageKey);
return readFile(absolutePath);
}
unlink(storageKey: string): Promise<void> {
const absolutePath = resolveStoragePath(storageKey);
return fsUnlink(absolutePath);
}
}
+1 -1
View File
@@ -14,7 +14,7 @@
"sourceMap": true, "sourceMap": true,
"outDir": "./dist", "outDir": "./dist",
"rootDir": "./src", "rootDir": "./src",
"types": ["node"], "types": ["node", "jest"],
"incremental": true, "incremental": true,
"skipLibCheck": true, "skipLibCheck": true,
"strictNullChecks": true, "strictNullChecks": true,
+1
View File
@@ -45,6 +45,7 @@ services:
- '${BACKEND_PORT}:${BACKEND_PORT}' - '${BACKEND_PORT}:${BACKEND_PORT}'
volumes: volumes:
- ./backend/src:/backend/src - ./backend/src:/backend/src
- ./backend/uploads:/backend/uploads
healthcheck: healthcheck:
test: ['CMD-SHELL', 'curl -f http://backend:${BACKEND_PORT}/api/health-check/'] test: ['CMD-SHELL', 'curl -f http://backend:${BACKEND_PORT}/api/health-check/']
interval: 1s interval: 1s