Slave/upload api #5

Merged
nobswebdev merged 9 commits from slave/upload-api into master 2026-10-12 20:12:16 +00:00
15 changed files with 412 additions and 92 deletions
Showing only changes of commit d4529376c6 - Show all commits
+2 -1
View File
@@ -2,6 +2,7 @@ import { Module } from '@nestjs/common';
import { StorageService } from './services/StorageService';
@Module({
providers: [StorageService]
providers: [StorageService],
exports: [StorageService]
})
export class StorageModule {}
+6 -1
View File
@@ -1,10 +1,15 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { StorageModule } from '../storage/StorageModule';
import { UploadController } from './controllers/UploadController';
import { Upload } from './entities/Upload';
import { UploadHotlink } from './entities/UploadHotlink';
import { UploadZk } from './entities/UploadZk';
import { UploadService } from './services/UploadService';
@Module({
imports: [TypeOrmModule.forFeature([Upload, UploadZk, UploadHotlink])]
imports: [TypeOrmModule.forFeature([Upload, UploadZk, UploadHotlink]), StorageModule],
controllers: [UploadController],
providers: [UploadService]
})
export class UploadModule {}
@@ -0,0 +1,23 @@
import { Body, Controller, Post } from '@nestjs/common';
import { Throttle } from '@nestjs/throttler';
import { throttleProfiles } from '../../../config/throttleProfiles';
import { CreateHotlinkUploadDto } from '../dto/CreateHotlinkUploadDto';
import { CreateZkUploadDto } from '../dto/CreateZkUploadDto';
import { UploadService } from '../services/UploadService';
@Controller('uploads')
export class UploadController {
constructor(private readonly uploadService: UploadService) {}
@Post('zk')
@Throttle(throttleProfiles.upload)
createZk(@Body() dto: CreateZkUploadDto) {
return this.uploadService.createZkUpload(dto);
}
@Post('hotlink')
@Throttle(throttleProfiles.upload)
createHotlink(@Body() dto: CreateHotlinkUploadDto) {
return this.uploadService.createHotlinkUpload(dto);
}
}
@@ -1,19 +1,9 @@
import { Type } from 'class-transformer';
import { ArrayMaxSize, ArrayMinSize, IsArray, IsNotEmpty, ValidateNested } from 'class-validator';
import { getAppConfig } from '../../../config';
import { CreateHotlinkUploadItemDto } from './CreateHotlinkUploadItemDto';
import { UploadBatchOptionsDto } from './UploadBatchOptionsDto';
import { IsMimeType, IsNotEmpty, IsString } from 'class-validator';
import { UploadCiphertextOptionsDto } from './UploadCiphertextOptionsDto';
const {
validation: { maxImagesPerUpload }
} = getAppConfig();
export class CreateHotlinkUploadDto extends UploadBatchOptionsDto {
export class CreateHotlinkUploadDto extends UploadCiphertextOptionsDto {
@IsNotEmpty()
@IsArray()
@ArrayMinSize(1)
@ArrayMaxSize(maxImagesPerUpload)
@ValidateNested({ each: true })
@Type(() => CreateHotlinkUploadItemDto)
items: CreateHotlinkUploadItemDto[];
@IsString()
@IsMimeType()
mimeType: string;
}
@@ -1,9 +0,0 @@
import { IsMimeType, IsNotEmpty, IsString } from 'class-validator';
import { CreateUploadCiphertextItemDto } from './CreateUploadCiphertextItemDto';
export class CreateHotlinkUploadItemDto extends CreateUploadCiphertextItemDto {
@IsNotEmpty()
@IsString()
@IsMimeType()
mimeType: string;
}
@@ -1,12 +0,0 @@
import { IsNotEmpty } from 'class-validator';
import { IsBase64 } from '../../../validation/decorators/isBase64';
export class CreateUploadCiphertextItemDto {
@IsNotEmpty()
@IsBase64({ byteLength: 12 })
iv: string;
@IsNotEmpty()
@IsBase64()
ciphertext: string;
}
@@ -1,17 +0,0 @@
import { IsNotEmpty } from 'class-validator';
import { IsBase64 } from '../../../validation/decorators/isBase64';
import { CreateUploadCiphertextItemDto } from './CreateUploadCiphertextItemDto';
export class CreateZkPasswordUploadItemDto extends CreateUploadCiphertextItemDto {
@IsNotEmpty()
@IsBase64()
wrappedDek: string;
@IsNotEmpty()
@IsBase64({ byteLength: 12 })
wrapIv: string;
@IsNotEmpty()
@IsBase64()
salt: string;
}
@@ -1,40 +1,24 @@
import { plainToInstance, Transform } from 'class-transformer';
import {
ArrayMaxSize,
ArrayMinSize,
IsArray,
IsBoolean,
IsNotEmpty,
ValidateNested
} from 'class-validator';
import { getAppConfig } from '../../../config';
import { CreateUploadCiphertextItemDto } from './CreateUploadCiphertextItemDto';
import { CreateZkPasswordUploadItemDto } from './CreateZkPasswordUploadItemDto';
import { UploadBatchOptionsDto } from './UploadBatchOptionsDto';
import { IsBoolean, IsNotEmpty, ValidateIf } from 'class-validator';
import { IsBase64 } from '../../../validation/decorators/isBase64';
import { UploadCiphertextOptionsDto } from './UploadCiphertextOptionsDto';
const {
validation: { maxImagesPerUpload }
} = getAppConfig();
export class CreateZkUploadDto extends UploadBatchOptionsDto {
export class CreateZkUploadDto extends UploadCiphertextOptionsDto {
@IsNotEmpty()
@IsBoolean()
passwordProtected: boolean;
@ValidateIf((dto: CreateZkUploadDto) => dto.passwordProtected === true)
@IsNotEmpty()
@IsArray()
@ArrayMinSize(1)
@ArrayMaxSize(maxImagesPerUpload)
@ValidateNested({ each: true })
@Transform(({ value, obj }: { value: unknown; obj: CreateZkUploadDto }) => {
if (!Array.isArray(value)) {
return value;
}
@IsBase64({ maxByteLength: 128 })
wrappedDek?: string;
const ItemClass =
obj.passwordProtected === true ? CreateZkPasswordUploadItemDto : CreateUploadCiphertextItemDto;
@ValidateIf((dto: CreateZkUploadDto) => dto.passwordProtected === true)
@IsNotEmpty()
@IsBase64({ byteLength: 12 })
wrapIv?: string;
return value.map(item => plainToInstance(ItemClass, item));
})
items: Array<CreateUploadCiphertextItemDto | CreateZkPasswordUploadItemDto>;
@ValidateIf((dto: CreateZkUploadDto) => dto.passwordProtected === true)
@IsNotEmpty()
@IsBase64({ maxByteLength: 64 })
salt?: string;
}
@@ -0,0 +1,20 @@
import { IsNotEmpty } from 'class-validator';
import { getAppConfig } from '../../../config';
import { IsBase64 } from '../../../validation/decorators/isBase64';
import { UploadOptionsDto } from './UploadOptionsDto';
const {
validation: {
upload: { maxCiphertextBytes }
}
} = getAppConfig();
export class UploadCiphertextOptionsDto extends UploadOptionsDto {
@IsNotEmpty()
@IsBase64({ byteLength: 12 })
iv: string;
@IsNotEmpty()
@IsBase64({ maxByteLength: maxCiphertextBytes })
ciphertext: string;
}
@@ -2,13 +2,15 @@ import { getAppConfig } from '../../../config';
import { NullOrDate, NullOrInt } from '../../../validation/decorators/nullOr';
const {
validation: { uploadMaxViewsMax }
validation: {
upload: { maxViewsMax }
}
} = getAppConfig();
export class UploadBatchOptionsDto {
export class UploadOptionsDto {
@NullOrDate()
expiresAt: Date | null;
@NullOrInt({ min: 1, max: uploadMaxViewsMax })
@NullOrInt({ min: 1, max: maxViewsMax })
maxViews: number | null;
}
@@ -26,10 +26,10 @@ export class Upload {
@Column()
deleteTokenHash: string;
@OneToOne(() => UploadZk, zk => zk.upload)
@OneToOne(() => UploadZk, zk => zk.upload, { cascade: true })
zk: UploadZk | null;
@OneToOne(() => UploadHotlink, hotlink => hotlink.upload)
@OneToOne(() => UploadHotlink, hotlink => hotlink.upload, { cascade: true })
hotlink: UploadHotlink | null;
@Column({ type: 'timestamptz', nullable: true })
@@ -0,0 +1,202 @@
import { BadRequestException } from '@nestjs/common';
import { createHash } from 'node:crypto';
import type { DataSource, EntityManager } from 'typeorm';
import { fileTypeFromBuffer } from 'file-type';
import { StorageService } from '../../storage/services/StorageService';
import { UploadMode } from '../types/UploadMode';
jest.mock('@nestjs/typeorm', () => ({
InjectDataSource: () => (): void => undefined
}));
jest.mock('file-type', () => ({
fileTypeFromBuffer: jest.fn()
}));
const fixedId = '11111111-1111-4111-8111-111111111111';
const fixedDeleteTokenBytes = Buffer.alloc(32, 0xab);
jest.mock('node:crypto', () => {
const actual = jest.requireActual<typeof import('node:crypto')>('node:crypto');
return {
...actual,
randomUUID: jest.fn(() => fixedId),
randomBytes: jest.fn((size: number) => (size === 32 ? fixedDeleteTokenBytes : Buffer.alloc(size)))
};
});
import { UploadService } from './UploadService';
const mockFileTypeFromBuffer = fileTypeFromBuffer as jest.MockedFunction<typeof fileTypeFromBuffer>;
const ivBase64 = (): string => Buffer.alloc(12, 0x11).toString('base64');
const ciphertextBase64 = (fill = 0xcd): string => Buffer.alloc(64, fill).toString('base64');
const wrapFieldBase64 = (length: number): string => Buffer.alloc(length, 0x22).toString('base64');
const hashDeleteToken = (token: string): string => createHash('sha256').update(token, 'utf8').digest('hex');
describe('UploadService', () => {
let service: UploadService;
let dataSource: jest.Mocked<Pick<DataSource, 'transaction'>>;
let storageService: jest.Mocked<Pick<StorageService, 'write'>>;
let uploadRepository: { create: jest.Mock; save: jest.Mock };
let transaction: jest.Mock;
beforeEach(() => {
uploadRepository = {
create: jest.fn((entityLike: unknown) => entityLike),
save: jest.fn(async (entity: unknown) => entity)
};
transaction = jest.fn(async (runInTransaction: (manager: EntityManager) => Promise<unknown>) => {
const manager = {
getRepository: jest.fn(() => uploadRepository)
} as unknown as EntityManager;
return runInTransaction(manager);
});
dataSource = { transaction };
storageService = { write: jest.fn().mockResolvedValue(undefined) };
mockFileTypeFromBuffer.mockResolvedValue(undefined);
service = new UploadService(
dataSource as unknown as DataSource,
storageService as unknown as StorageService
);
});
describe('createHotlinkUpload', () => {
it('persists upload with hotlink relation and writes ciphertext', async () => {
const ciphertext = ciphertextBase64();
const ciphertextBuffer = Buffer.from(ciphertext, 'base64');
const iv = ivBase64();
const deleteToken = fixedDeleteTokenBytes.toString('base64url');
const result = await service.createHotlinkUpload({
iv,
ciphertext,
expiresAt: null,
maxViews: null,
mimeType: 'image/png'
});
expect(result).toEqual({
id: fixedId,
mode: UploadMode.Hotlink,
deleteToken
});
expect(uploadRepository.create).toHaveBeenCalledWith({
id: fixedId,
mode: UploadMode.Hotlink,
storagePath: fixedId,
iv: Buffer.from(iv, 'base64'),
maxViews: null,
deleteTokenHash: hashDeleteToken(deleteToken),
expiresAt: null,
hotlink: { mimeType: 'image/png' }
});
expect(uploadRepository.save).toHaveBeenCalledTimes(1);
expect(storageService.write).toHaveBeenCalledWith(fixedId, ciphertextBuffer);
expect(transaction).toHaveBeenCalledTimes(1);
});
});
describe('createZkUpload', () => {
it('persists zk upload without password wrap fields when not password protected', async () => {
const ciphertext = ciphertextBase64();
await service.createZkUpload({
iv: ivBase64(),
ciphertext,
expiresAt: null,
maxViews: 5,
passwordProtected: false
});
expect(uploadRepository.create).toHaveBeenCalledWith(
expect.objectContaining({
mode: UploadMode.Zk,
maxViews: 5,
zk: {
passwordProtected: false,
wrappedDek: null,
wrapIv: null,
salt: null
}
})
);
expect(storageService.write).toHaveBeenCalledWith(fixedId, Buffer.from(ciphertext, 'base64'));
});
it('decodes password wrap fields when password protected', async () => {
const wrappedDek = wrapFieldBase64(32);
const wrapIv = ivBase64();
const salt = wrapFieldBase64(16);
await service.createZkUpload({
iv: ivBase64(),
ciphertext: ciphertextBase64(),
expiresAt: null,
maxViews: null,
passwordProtected: true,
wrappedDek,
wrapIv,
salt
});
expect(uploadRepository.create).toHaveBeenCalledWith(
expect.objectContaining({
zk: {
passwordProtected: true,
wrappedDek: Buffer.from(wrappedDek, 'base64'),
wrapIv: Buffer.from(wrapIv, 'base64'),
salt: Buffer.from(salt, 'base64')
}
})
);
});
});
describe('throwIfUnencryptedImage', () => {
it('rejects ciphertext that sniffs as an image before persisting', async () => {
mockFileTypeFromBuffer.mockResolvedValue({ ext: 'jpg', mime: 'image/jpeg' });
await expect(
service.createHotlinkUpload({
iv: ivBase64(),
ciphertext: ciphertextBase64(0xff),
expiresAt: null,
maxViews: null,
mimeType: 'image/jpeg'
})
).rejects.toThrow(BadRequestException);
expect(transaction).not.toHaveBeenCalled();
expect(storageService.write).not.toHaveBeenCalled();
});
});
describe('transaction rollback on storage failure', () => {
it('propagates storage write errors from the transaction', async () => {
storageService.write.mockRejectedValue(new Error('disk full'));
await expect(
service.createHotlinkUpload({
iv: ivBase64(),
ciphertext: ciphertextBase64(),
expiresAt: null,
maxViews: null,
mimeType: 'image/png'
})
).rejects.toThrow('disk full');
expect(uploadRepository.save).toHaveBeenCalledTimes(1);
});
});
});
@@ -0,0 +1,123 @@
import { BadRequestException, Injectable } from '@nestjs/common';
import { InjectDataSource } from '@nestjs/typeorm';
import { fileTypeFromBuffer } from 'file-type';
import { createHash, randomBytes, randomUUID } from 'node:crypto';
import { DataSource } from 'typeorm';
import { fromBase64 } from '../../../utils/fromBase64';
import { StorageService } from '../../storage/services/StorageService';
import { CreateHotlinkUploadDto } from '../dto/CreateHotlinkUploadDto';
import { CreateZkUploadDto } from '../dto/CreateZkUploadDto';
import { Upload } from '../entities/Upload';
import { CreatedUploadResult } from '../types/CreatedUploadResult';
import { UploadMode } from '../types/UploadMode';
@Injectable()
export class UploadService {
constructor(
@InjectDataSource()
private readonly dataSource: DataSource,
private readonly storageService: StorageService
) {}
async createZkUpload({
iv,
ciphertext,
expiresAt,
maxViews,
passwordProtected,
wrappedDek,
wrapIv,
salt
}: CreateZkUploadDto): Promise<CreatedUploadResult> {
const ciphertextBuffer = fromBase64(ciphertext);
await this.throwIfUnencryptedImage(ciphertextBuffer);
const deleteToken = randomBytes(32).toString('base64url');
const id = randomUUID();
const storagePath = id;
await this.dataSource.transaction(async manager => {
const uploadRepository = manager.getRepository(Upload);
const upload = uploadRepository.create({
id,
mode: UploadMode.Zk,
storagePath,
iv: fromBase64(iv),
maxViews,
deleteTokenHash: this.hashDeleteToken(deleteToken),
expiresAt,
zk: {
passwordProtected,
wrappedDek: passwordProtected && wrappedDek ? fromBase64(wrappedDek) : null,
wrapIv: passwordProtected && wrapIv ? fromBase64(wrapIv) : null,
salt: passwordProtected && salt ? fromBase64(salt) : null
}
});
await uploadRepository.save(upload);
await this.storageService.write(storagePath, ciphertextBuffer);
});
return {
id,
mode: UploadMode.Zk,
deleteToken
};
}
async createHotlinkUpload({
iv,
ciphertext,
expiresAt,
maxViews,
mimeType
}: CreateHotlinkUploadDto): Promise<CreatedUploadResult> {
const ciphertextBuffer = fromBase64(ciphertext);
await this.throwIfUnencryptedImage(ciphertextBuffer);
const deleteToken = randomBytes(32).toString('base64url');
const id = randomUUID();
const storagePath = id;
await this.dataSource.transaction(async manager => {
const uploadRepository = manager.getRepository(Upload);
const upload = uploadRepository.create({
id,
mode: UploadMode.Hotlink,
storagePath,
iv: fromBase64(iv),
maxViews,
deleteTokenHash: this.hashDeleteToken(deleteToken),
expiresAt,
hotlink: { mimeType }
});
await uploadRepository.save(upload);
await this.storageService.write(storagePath, ciphertextBuffer);
});
return {
id,
mode: UploadMode.Hotlink,
deleteToken
};
}
private hashDeleteToken(token: string): string {
return createHash('sha256').update(token, 'utf8').digest('hex');
}
private async throwIfUnencryptedImage(ciphertext: Buffer): Promise<void> {
const detected = await fileTypeFromBuffer(ciphertext);
if (detected?.mime.startsWith('image/')) {
throw new BadRequestException('Ciphertext must not be a plaintext image');
}
}
}
@@ -0,0 +1,7 @@
import { UploadMode } from './UploadMode';
export type CreatedUploadResult = {
id: string;
mode: UploadMode;
deleteToken: string;
};
+1
View File
@@ -0,0 +1 @@
export const fromBase64 = (value: string): Buffer => Buffer.from(value, 'base64');