Merge pull request 'Slave/db storage' (#3) from slave/db-storage into master

Reviewed-on: #3

.env.example

@@ -5,6 +5,7 @@ POSTGRES_DB=private_image
 POSTGRES_USER=postgres
 POSTGRES_PASSWORD=postgres
 POSTGRES_PORT=5432
+POSTGRES_MIGRATIONS_RUN=false
 
 PGADMIN_DEFAULT_EMAIL=admin@example.com
 PGADMIN_DEFAULT_PASSWORD=admin

.gitignore

@@ -7,7 +7,7 @@
 /backend/node_modules
 /backend/dist
 /backend/*.tsbuildinfo
-/backend/storage
+/backend/uploads
 
 /frontend/node_modules
 /frontend/dist

backend/package.json

@@ -7,22 +7,41 @@
         "build": "nest build",
         "start": "nest start",
         "start:dev": "nest start --watch",
-        "start:prod": "node dist/main"
+        "start:prod": "node dist/main",
+        "typeorm": "ts-node -r tsconfig-paths/register ./node_modules/typeorm/cli",
+        "typeorm:run-migrations": "npm run typeorm migration:run -- -d ./src/database/DataSource.ts",
+        "typeorm:generate-migration": "npm run typeorm -- -d ./src/database/DataSource.ts migration:generate ./src/database/migrations/$npm_config_name",
+        "typeorm:revert-migration": "npm run typeorm -- -d ./src/database/DataSource.ts migration:revert",
+        "typeorm:create-migration": "npm run typeorm -- migration:create ./src/database/migrations/$npm_config_name",
+        "test": "jest"
+    },
+    "jest": {
+        "preset": "ts-jest",
+        "rootDir": "src"
     },
     "dependencies": {
         "@nestjs/common": "^11.0.1",
         "@nestjs/config": "^4.0.3",
         "@nestjs/core": "^11.0.1",
         "@nestjs/platform-express": "^11.0.1",
+        "@nestjs/typeorm": "^12.0.2",
         "class-transformer": "^0.5.1",
         "class-validator": "^0.15.1",
-        "reflect-metadata": "^0.2.2"
+        "pg": "^8.23.1",
+        "reflect-metadata": "^0.2.2",
+        "typeorm": "^0.3.28"
     },
     "devDependencies": {
         "@nestjs/cli": "^11.0.24",
         "@nestjs/schematics": "^11.0.0",
         "@types/express": "^5.0.0",
+        "@types/jest": "^30.0.0",
         "@types/node": "^22.10.7",
+        "@types/pg": "^8.23.1",
+        "jest": "^30.5.2",
+        "ts-jest": "^29.4.14",
+        "ts-node": "^10.9.2",
+        "tsconfig-paths": "^4.2.0",
         "typescript": "~6.0.2"
     }
 }

backend/src/AppModule.ts

@@ -1,15 +1,25 @@
 import { Module } from '@nestjs/common';
-import { ConfigModule } from '@nestjs/config';
-import appConfig from './config';
+import { ConfigModule, ConfigService, registerAs } from '@nestjs/config';
+import { TypeOrmModule } from '@nestjs/typeorm';
+import { getAppConfig, getPostgresConfig } from './config';
 import { HealthCheckModule } from './modules/healthCheck/HealthCheckModule';
+import { StorageModule } from './modules/storage/StorageModule';
+import { UploadModule } from './modules/upload/UploadModule';
+import { Config } from './types/Config';
 
 @Module({
     imports: [
         ConfigModule.forRoot({
             isGlobal: true,
-            load: [appConfig]
+            load: [registerAs('postgres', getPostgresConfig), registerAs('app', getAppConfig)]
         }),
-        HealthCheckModule
+        TypeOrmModule.forRootAsync({
+            useFactory: (configService: ConfigService) => configService.get('postgres') as Config['postgres'],
+            inject: [ConfigService]
+        }),
+        HealthCheckModule,
+        StorageModule,
+        UploadModule
     ]
 })
 export class AppModule {}

backend/src/config/index.ts

@@ -1,11 +1,28 @@
-import { registerAs } from '@nestjs/config';
 import { AppConfig } from '../types/AppConfig';
+import { PostgresConfig } from '../types/Config';
 import { NodeEnv } from '../types/NodeEnv';
 
 const env = (key: string): string => process.env[key] || '';
 
 const envInt = (key: string): number => parseInt(env(key), 10);
 
+const isEnabled = (key: string): boolean => env(key) === 'true';
+
+export const getPostgresConfig = (): PostgresConfig => {
+    return {
+        type: 'postgres',
+        url: `postgres://${env('POSTGRES_USER')}:${env('POSTGRES_PASSWORD')}@${env('POSTGRES_HOST')}:${env('POSTGRES_PORT')}/${env('POSTGRES_DB')}`,
+        host: env('POSTGRES_HOST'),
+        username: env('POSTGRES_USER'),
+        password: env('POSTGRES_PASSWORD'),
+        port: envInt('POSTGRES_PORT'),
+        database: env('POSTGRES_DB'),
+        entities: [__dirname + '/../modules/**/entities/*{.ts,.js}'],
+        migrations: [__dirname + '/../database/migrations/*{.ts,.js}'],
+        migrationsRun: isEnabled('POSTGRES_MIGRATIONS_RUN')
+    };
+};
+
 export const getAppConfig = (): AppConfig => {
     const rawOrigins = env('CORS_ORIGINS');
 
@@ -18,5 +35,3 @@ export const getAppConfig = (): AppConfig => {
             .filter(Boolean)
     };
 };
-
-export default registerAs('app', getAppConfig);

backend/src/config/uploadPaths.spec.ts

@@ -0,0 +1,61 @@
+import { join, resolve } from 'node:path';
+import { getUploadsDir, resolveStoragePath } from './uploadPaths';
+
+describe('uploadPaths', () => {
+    describe('getUploadsDir', () => {
+        it('returns uploads under process cwd', () => {
+            expect(getUploadsDir()).toBe(join(process.cwd(), 'uploads'));
+        });
+
+        it('returns a stable path on repeated calls', () => {
+            expect(getUploadsDir()).toBe(getUploadsDir());
+        });
+    });
+
+    describe('resolveStoragePath', () => {
+        it('resolves a storage key under uploads dir', () => {
+            const storageKey = 'a1b2c3d4-e5f6-7890-abcd-ef1234567890';
+
+            expect(resolveStoragePath(storageKey)).toBe(resolve(getUploadsDir(), storageKey));
+        });
+
+        it('resolves a nested storage key under uploads dir', () => {
+            const storageKey = 'nested/a1b2c3d4-e5f6-7890-abcd-ef1234567890';
+
+            expect(resolveStoragePath(storageKey)).toBe(resolve(getUploadsDir(), storageKey));
+        });
+
+        it('rejects path traversal with parent segments', () => {
+            expect(() => resolveStoragePath('../../etc/passwd')).toThrow('Invalid storage key');
+            expect(() => resolveStoragePath('../outside')).toThrow('Invalid storage key');
+        });
+
+        it('rejects absolute storage keys outside uploads dir', () => {
+            expect(() => resolveStoragePath('/etc/passwd')).toThrow('Invalid storage key');
+        });
+
+        it('rejects empty storage key', () => {
+            expect(() => resolveStoragePath('')).toThrow('Invalid storage key');
+        });
+
+        it('rejects traversal hidden after a benign prefix', () => {
+            expect(() => resolveStoragePath('safe/../../etc/passwd')).toThrow('Invalid storage key');
+        });
+
+        it.each(['..', 'foo/../../../etc/passwd'])('rejects suspicious storage key: %s', storageKey => {
+            expect(() => resolveStoragePath(storageKey)).toThrow('Invalid storage key');
+        });
+
+        it('treats encoded traversal literals as opaque file names', () => {
+            const storageKey = '%2e%2e%2fetc/passwd';
+
+            expect(resolveStoragePath(storageKey)).toBe(resolve(getUploadsDir(), storageKey));
+        });
+
+        it('allows dot characters inside a single file name segment', () => {
+            const storageKey = 'my.file.name.bin';
+
+            expect(resolveStoragePath(storageKey)).toBe(resolve(getUploadsDir(), storageKey));
+        });
+    });
+});

backend/src/config/uploadPaths.ts

@@ -0,0 +1,26 @@
+import { join, resolve, sep } from 'node:path';
+
+export const getUploadsDir = (): string => join(process.cwd(), 'uploads');
+
+const validatePathTraversal = (baseDir: string, absolutePath: string): void => {
+    const resolvedBaseDir = resolve(baseDir);
+
+    if (!absolutePath.startsWith(`${resolvedBaseDir}${sep}`)) {
+        throw new Error('Invalid storage key');
+    }
+};
+
+const resolvePathWithinDir = (baseDir: string, storageKey: string): string => {
+    const resolvedBaseDir = resolve(baseDir);
+    const absolutePath = resolve(resolvedBaseDir, storageKey);
+
+    validatePathTraversal(resolvedBaseDir, absolutePath);
+
+    return absolutePath;
+};
+
+export const resolveStoragePath = (storageKey: string): string => {
+    const uploadsDir = getUploadsDir();
+
+    return resolvePathWithinDir(uploadsDir, storageKey);
+};

backend/src/database/DataSource.ts

@@ -0,0 +1,5 @@
+import { DataSource } from 'typeorm';
+
+import { getPostgresConfig } from '../config';
+
+export default new DataSource(getPostgresConfig());

backend/src/database/migrations/1791230839246-add-upload-table.ts

@@ -0,0 +1,33 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class AddUploadTable1791230839246 implements MigrationInterface {
+    name = 'AddUploadTable1791230839246';
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(
+            `CREATE TABLE "upload_zk" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "passwordProtected" boolean NOT NULL DEFAULT false, "wrappedDek" bytea, "wrapIv" bytea, "salt" bytea, "uploadId" uuid, CONSTRAINT "REL_690bb5d62da9aed77980cb1106" UNIQUE ("uploadId"), CONSTRAINT "PK_8e9c09159715a88387bc8a1fa4a" PRIMARY KEY ("id"))`
+        );
+        await queryRunner.query(`CREATE TYPE "public"."uploads_mode_enum" AS ENUM('zk', 'hotlink')`);
+        await queryRunner.query(
+            `CREATE TABLE "uploads" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "mode" "public"."uploads_mode_enum" NOT NULL, "storagePath" text NOT NULL, "iv" bytea NOT NULL, "maxViews" integer, "viewCount" integer NOT NULL DEFAULT '0', "lastViewedAt" TIMESTAMP WITH TIME ZONE, "deleteTokenHash" character varying NOT NULL, "expiresAt" TIMESTAMP WITH TIME ZONE, "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), CONSTRAINT "PK_d1781d1eedd7459314f60f39bd3" PRIMARY KEY ("id"))`
+        );
+        await queryRunner.query(
+            `CREATE TABLE "upload_hotlink" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "mimeType" character varying NOT NULL, "uploadId" uuid, CONSTRAINT "REL_05156f9756799f6aecfcc14c10" UNIQUE ("uploadId"), CONSTRAINT "PK_abe0f3d297ad4c63773135691a4" PRIMARY KEY ("id"))`
+        );
+        await queryRunner.query(
+            `ALTER TABLE "upload_zk" ADD CONSTRAINT "FK_690bb5d62da9aed77980cb11067" FOREIGN KEY ("uploadId") REFERENCES "uploads"("id") ON DELETE CASCADE ON UPDATE NO ACTION`
+        );
+        await queryRunner.query(
+            `ALTER TABLE "upload_hotlink" ADD CONSTRAINT "FK_05156f9756799f6aecfcc14c108" FOREIGN KEY ("uploadId") REFERENCES "uploads"("id") ON DELETE CASCADE ON UPDATE NO ACTION`
+        );
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "upload_hotlink" DROP CONSTRAINT "FK_05156f9756799f6aecfcc14c108"`);
+        await queryRunner.query(`ALTER TABLE "upload_zk" DROP CONSTRAINT "FK_690bb5d62da9aed77980cb11067"`);
+        await queryRunner.query(`DROP TABLE "upload_hotlink"`);
+        await queryRunner.query(`DROP TABLE "uploads"`);
+        await queryRunner.query(`DROP TYPE "public"."uploads_mode_enum"`);
+        await queryRunner.query(`DROP TABLE "upload_zk"`);
+    }
+}

backend/src/main.ts

@@ -2,13 +2,13 @@ import { ValidationPipe } from '@nestjs/common';
 import { ConfigService } from '@nestjs/config';
 import { NestFactory } from '@nestjs/core';
 import { AppModule } from './AppModule';
-import { AppConfig } from './types/AppConfig';
+import { Config } from './types/Config';
 
 async function bootstrap() {
     const app = await NestFactory.create(AppModule);
 
     const configService = app.get(ConfigService);
-    const { port, corsOrigins } = configService.get('app') as AppConfig;
+    const { port, corsOrigins } = configService.get('app') as Config['app'];
 
     app.setGlobalPrefix('api');
 

backend/src/modules/storage/StorageModule.ts

@@ -0,0 +1,8 @@
+import { Module } from '@nestjs/common';
+import { StorageService } from './services/StorageService';
+
+@Module({
+    providers: [StorageService],
+    exports: [StorageService]
+})
+export class StorageModule {}

backend/src/modules/storage/services/StorageService.spec.ts

@@ -0,0 +1,120 @@
+import { access, mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { getUploadsDir, resolveStoragePath } from '../../../config/uploadPaths';
+import { StorageService } from './StorageService';
+
+describe('StorageService', () => {
+    let service: StorageService;
+    let previousCwd: string;
+    let tempRoot: string;
+
+    beforeEach(async () => {
+        service = new StorageService();
+        previousCwd = process.cwd();
+        tempRoot = await mkdtemp(join(tmpdir(), 'storage-service-'));
+        process.chdir(tempRoot);
+    });
+
+    afterEach(async () => {
+        process.chdir(previousCwd);
+        await rm(tempRoot, { recursive: true, force: true });
+    });
+
+    it('writes and reads ciphertext bytes', async () => {
+        const storageKey = 'a1b2c3d4-e5f6-7890-abcd-ef1234567890';
+        const data = Buffer.from('ciphertext-bytes');
+
+        await service.write(storageKey, data);
+
+        expect(await service.read(storageKey)).toEqual(data);
+    });
+
+    it('creates the uploads directory on write', async () => {
+        await service.write('file-id', Buffer.from('x'));
+
+        await expect(access(getUploadsDir())).resolves.toBeUndefined();
+    });
+
+    it('writes nested storage keys under uploads', async () => {
+        const storageKey = 'nested/file-id';
+        const data = Buffer.from('nested');
+
+        await service.write(storageKey, data);
+
+        expect(await service.read(storageKey)).toEqual(data);
+    });
+
+    it('unlinks a stored file', async () => {
+        const storageKey = 'to-delete';
+
+        await service.write(storageKey, Buffer.from('delete-me'));
+        await service.unlink(storageKey);
+
+        await expect(access(resolveStoragePath(storageKey))).rejects.toMatchObject({ code: 'ENOENT' });
+    });
+
+    it.each(['../../escape', '../outside', '/etc/passwd', ''])(
+        'rejects invalid storage key on write: %s',
+        async storageKey => {
+            await expect(service.write(storageKey, Buffer.from('x'))).rejects.toThrow('Invalid storage key');
+        }
+    );
+
+    it.each(['../../escape', '../outside', '/etc/passwd', ''])(
+        'rejects invalid storage key on read: %s',
+        storageKey => {
+            expect(() => service.read(storageKey)).toThrow('Invalid storage key');
+        }
+    );
+
+    it.each(['../../escape', '../outside', '/etc/passwd', ''])(
+        'rejects invalid storage key on unlink: %s',
+        storageKey => {
+            expect(() => service.unlink(storageKey)).toThrow('Invalid storage key');
+        }
+    );
+
+    it('throws when reading a missing file', async () => {
+        await expect(service.read('missing-id')).rejects.toMatchObject({ code: 'ENOENT' });
+    });
+
+    it('throws when unlinking a missing file', async () => {
+        await expect(service.unlink('missing-id')).rejects.toMatchObject({ code: 'ENOENT' });
+    });
+
+    it('overwrites an existing storage key on second write', async () => {
+        const storageKey = 'overwrite-me';
+
+        await service.write(storageKey, Buffer.from('first'));
+        await service.write(storageKey, Buffer.from('second'));
+
+        expect(await service.read(storageKey)).toEqual(Buffer.from('second'));
+    });
+
+    it('keeps separate storage keys isolated', async () => {
+        await service.write('key-a', Buffer.from('a'));
+        await service.write('key-b', Buffer.from('b'));
+
+        expect(await service.read('key-a')).toEqual(Buffer.from('a'));
+        expect(await service.read('key-b')).toEqual(Buffer.from('b'));
+    });
+
+    it('round-trips an empty buffer', async () => {
+        const storageKey = 'empty';
+        const data = Buffer.alloc(0);
+
+        await service.write(storageKey, data);
+
+        expect(await service.read(storageKey)).toEqual(data);
+    });
+
+    it('round-trips binary ciphertext bytes', async () => {
+        const storageKey = 'binary';
+        const data = Buffer.from([0x00, 0xff, 0x10, 0x89, 0xab]);
+
+        await service.write(storageKey, data);
+
+        expect(await service.read(storageKey)).toEqual(data);
+    });
+});

backend/src/modules/storage/services/StorageService.ts

@@ -0,0 +1,28 @@
+import { mkdir, readFile, unlink as fsUnlink, writeFile } from 'node:fs/promises';
+import { dirname } from 'node:path';
+import { Injectable } from '@nestjs/common';
+import { resolveStoragePath } from '../../../config/uploadPaths';
+
+@Injectable()
+export class StorageService {
+    async write(storageKey: string, data: Buffer): Promise<void> {
+        const absolutePath = resolveStoragePath(storageKey);
+        const dir = dirname(absolutePath);
+
+        await mkdir(dir, { recursive: true });
+
+        return writeFile(absolutePath, data);
+    }
+
+    read(storageKey: string): Promise<Buffer> {
+        const absolutePath = resolveStoragePath(storageKey);
+
+        return readFile(absolutePath);
+    }
+
+    unlink(storageKey: string): Promise<void> {
+        const absolutePath = resolveStoragePath(storageKey);
+
+        return fsUnlink(absolutePath);
+    }
+}

backend/src/modules/upload/UploadModule.ts

@@ -0,0 +1,10 @@
+import { Module } from '@nestjs/common';
+import { TypeOrmModule } from '@nestjs/typeorm';
+import { Upload } from './entities/Upload';
+import { UploadHotlink } from './entities/UploadHotlink';
+import { UploadZk } from './entities/UploadZk';
+
+@Module({
+    imports: [TypeOrmModule.forFeature([Upload, UploadZk, UploadHotlink])]
+})
+export class UploadModule {}

backend/src/modules/upload/entities/Upload.ts

@@ -0,0 +1,43 @@
+import { Column, CreateDateColumn, Entity, OneToOne, PrimaryGeneratedColumn } from 'typeorm';
+import { UploadMode } from '../types/UploadMode';
+import { UploadHotlink } from './UploadHotlink';
+import { UploadZk } from './UploadZk';
+
+@Entity('uploads')
+export class Upload {
+    @PrimaryGeneratedColumn('uuid')
+    id: string;
+
+    @Column({ type: 'enum', enum: UploadMode })
+    mode: UploadMode;
+
+    @Column({ type: 'text' })
+    storagePath: string;
+
+    @Column({ type: 'bytea' })
+    iv: Buffer;
+
+    @Column({ type: 'int', nullable: true })
+    maxViews: number | null;
+
+    @Column({ type: 'int', default: 0 })
+    viewCount: number;
+
+    @Column()
+    deleteTokenHash: string;
+
+    @OneToOne(() => UploadZk, zk => zk.upload)
+    zk: UploadZk | null;
+
+    @OneToOne(() => UploadHotlink, hotlink => hotlink.upload)
+    hotlink: UploadHotlink | null;
+
+    @Column({ type: 'timestamptz', nullable: true })
+    lastViewedAt: Date | null;
+
+    @Column({ type: 'timestamptz', nullable: true })
+    expiresAt: Date | null;
+
+    @CreateDateColumn({ type: 'timestamptz' })
+    createdAt: Date;
+}

backend/src/modules/upload/entities/UploadHotlink.ts

@@ -0,0 +1,15 @@
+import { Column, Entity, JoinColumn, OneToOne, PrimaryGeneratedColumn } from 'typeorm';
+import { Upload } from './Upload';
+
+@Entity('upload_hotlink')
+export class UploadHotlink {
+    @PrimaryGeneratedColumn('uuid')
+    id: string;
+
+    @Column()
+    mimeType: string;
+
+    @OneToOne(() => Upload, upload => upload.hotlink, { onDelete: 'CASCADE' })
+    @JoinColumn()
+    upload: Upload;
+}

backend/src/modules/upload/entities/UploadZk.ts

@@ -0,0 +1,24 @@
+import { Column, Entity, JoinColumn, OneToOne, PrimaryGeneratedColumn } from 'typeorm';
+import { Upload } from './Upload';
+
+@Entity('upload_zk')
+export class UploadZk {
+    @PrimaryGeneratedColumn('uuid')
+    id: string;
+
+    @Column({ type: 'boolean', default: false })
+    passwordProtected: boolean;
+
+    @Column({ type: 'bytea', nullable: true })
+    wrappedDek: Buffer | null;
+
+    @Column({ type: 'bytea', nullable: true })
+    wrapIv: Buffer | null;
+
+    @Column({ type: 'bytea', nullable: true })
+    salt: Buffer | null;
+
+    @OneToOne(() => Upload, upload => upload.zk, { onDelete: 'CASCADE' })
+    @JoinColumn()
+    upload: Upload;
+}

backend/src/modules/upload/types/UploadMode.ts

@@ -0,0 +1,4 @@
+export enum UploadMode {
+    Zk = 'zk',
+    Hotlink = 'hotlink'
+}

backend/src/types/Config.ts

@@ -0,0 +1,19 @@
+import { AppConfig } from './AppConfig';
+
+export interface PostgresConfig {
+    type: 'postgres';
+    url: string;
+    host: string;
+    username: string;
+    password: string;
+    port: number;
+    database: string;
+    entities: string[];
+    migrations: string[];
+    migrationsRun: boolean;
+}
+
+export interface Config {
+    app: AppConfig;
+    postgres: PostgresConfig;
+}

backend/tsconfig.json

@@ -14,7 +14,7 @@
         "sourceMap": true,
         "outDir": "./dist",
         "rootDir": "./src",
-        "types": ["node"],
+        "types": ["node", "jest"],
         "incremental": true,
         "skipLibCheck": true,
         "strictNullChecks": true,

docker-compose.dev.yml

@@ -45,6 +45,7 @@ services:
             - '${BACKEND_PORT}:${BACKEND_PORT}'
         volumes:
             - ./backend/src:/backend/src
+            - ./backend/uploads:/backend/uploads
         healthcheck:
             test: ['CMD-SHELL', 'curl -f http://backend:${BACKEND_PORT}/api/health-check/']
             interval: 1s