Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions src/services/drive/drive-file.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ export class DriveFileService {
itemType: 'file',
name: payload.plainName,
uuid: driveFile.uuid,
size: driveFile.size,
size: DriveUtils.parseFileSize(driveFile.size),
bucket: driveFile.bucket,
createdAt: new Date(driveFile.createdAt),
updatedAt: new Date(driveFile.updatedAt),
Expand All @@ -41,7 +41,7 @@ export class DriveFileService {
itemType: 'file',
name: driveFile.plainName ?? '',
uuid: driveFile.uuid,
size: driveFile.size,
size: DriveUtils.parseFileSize(driveFile.size),
bucket: driveFile.bucket,
createdAt: new Date(driveFile.createdAt),
updatedAt: new Date(driveFile.updatedAt),
Expand All @@ -66,7 +66,8 @@ export class DriveFileService {
filesList: [file],
});

const existentFile = existentFiles[0];
// The duplicate check can match a file with the same name but a different extension
const existentFile = existentFiles.find((existent) => (existent.type ?? '') === (file.type ?? ''));
if (!existentFile) return undefined;

return {
Expand Down
58 changes: 58 additions & 0 deletions src/services/drive/drive-item.service.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import path from 'node:path';
import { DriveItemRepository } from '../database/drive-item/drive-item.repository';
import { DriveFileItem, DriveFolderItem } from '../../types/drive.types';
import { ErrorUtils } from '../../utils/errors.utils';
Expand All @@ -6,9 +7,60 @@ import { DriveFileService } from './drive-file.service';
import { DriveFolderService } from './drive-folder.service';
import { DriveItemBD } from '../database/drive-item/drive-item.domain';

const RECENT_WRITE_TRUST_MS = 10_000;

export class DriveItemService {
static readonly instance = new DriveItemService();

private readonly recentFileWrites = new Map<string, { item: DriveFileItem; expiresAt: number }>();

/** The API reads from replicas that can lag a moment behind a write, so a file this server
* just created can briefly look missing; remember it so that 404 is not trusted. */
public rememberFileWrite = (path: string, item: DriveFileItem): void => {
const now = Date.now();
for (const [writtenPath, write] of this.recentFileWrites) {
if (write.expiresAt <= now) this.recentFileWrites.delete(writtenPath);
}
this.recentFileWrites.set(path, { item, expiresAt: now + RECENT_WRITE_TRUST_MS });
};

public moveFileWrite = (fromPath: string, toPath: string): void => {
const write = this.recentFileWrites.get(fromPath);
if (!write) return;
this.recentFileWrites.delete(fromPath);
const { name, ext } = path.parse(toPath);
this.recentFileWrites.set(toPath, { ...write, item: { ...write.item, name, type: ext.replace('.', '') } });
};

public forgetFileWrite = (itemPath: string, uuid: string): void => {
if (this.recentFileWrites.get(itemPath)?.item.uuid === uuid) this.recentFileWrites.delete(itemPath);
};

/** Forgets writes at this path and, for a folder, everything under it. */
public forgetFileWrites = (itemPath: string): void => {
const folderPrefix = itemPath.endsWith('/') ? itemPath : `${itemPath}/`;
for (const writtenPath of this.recentFileWrites.keys()) {
if (writtenPath === itemPath || writtenPath.startsWith(folderPrefix)) this.recentFileWrites.delete(writtenPath);
}
};

/** Only while the path cache still maps to the written uuid: our own DELETE and MOVE clear it. */
private readonly getRecentFileWrite = (
cached: DriveItemBD | undefined,
path: string,
error: unknown,
): DriveFileItem | undefined => {
if (!cached || !ErrorUtils.isNotFoundError(error)) return;

const write = this.recentFileWrites.get(path);
if (!write || write.expiresAt <= Date.now() || write.item.uuid !== cached.uuid) return;

webdavLogger.info(`[LOOKUP] '${path}' was written moments ago, ignoring the API's not found`, {
uuid: cached.uuid,
});
return write.item;
};

/** Only drop the cached uuid when the API confirms the item is gone; evicting on a timeout
* pushes every later request onto the slow path lookup. */
private readonly dropCacheIfGone = async (uuid: string, error: unknown): Promise<void> => {
Expand All @@ -30,6 +82,9 @@ export class DriveItemService {
]);
return item;
} catch (error) {
const recentWrite = this.getRecentFileWrite(cached, path, error);
if (recentWrite) return recentWrite;

webdavLogger.warn(ErrorUtils.withRequestId('File metadata by uuid failed, falling back to path lookup', error), {
path,
uuid: cached.uuid,
Expand Down Expand Up @@ -84,6 +139,9 @@ export class DriveItemService {
]);
return item;
} catch (error) {
const recentWrite = this.getRecentFileWrite(cached, path, error);
if (recentWrite) return recentWrite;

ErrorUtils.logIfUnexpected(webdavLogger, 'File lookup by path failed', error, { path });
throw ErrorUtils.toLookupError('file', path, error);
}
Expand Down
10 changes: 10 additions & 0 deletions src/utils/errors.utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ export const DEFAULT_RETRY_AFTER_SECONDS = 5;

export const CLIENT_VERSION_UNSUPPORTED = 'CLIENT_VERSION_UNSUPPORTED';

export const FILE_NOT_REPLACEABLE = 'FILE_NOT_REPLACEABLE';

export class ErrorUtils {
static readonly isError = (error: unknown): error is Error => {
return typeof Error.isError === 'function'
Expand Down Expand Up @@ -38,6 +40,10 @@ export class ErrorUtils {
);
};

static readonly isApiConflict = (error: unknown): boolean => {
return this.getStatusCode(error, 'status') === 409;
};

static readonly isFileNotFoundError = (error: unknown): error is NodeJS.ErrnoException => {
return this.isError(error) && 'code' in error && error.code === 'ENOENT';
};
Expand Down Expand Up @@ -93,6 +99,10 @@ export class ErrorUtils {
return this.getResponseBody(error)?.errorCode === CLIENT_VERSION_UNSUPPORTED;
};

static readonly isFileNotReplaceable = (error: unknown): boolean => {
return this.getResponseBody(error)?.errorCode === FILE_NOT_REPLACEABLE;
};

static readonly toClientVersionUnsupportedError = (error: unknown): ClientVersionUnsupportedError => {
if (error instanceof ClientVersionUnsupportedError) return error;

Expand Down
2 changes: 2 additions & 0 deletions src/webdav/handlers/DELETE.handler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { WebDavMethodHandler } from '../../types/webdav.types';
import { WebDavUtils } from '../../utils/webdav.utils';
import { webdavLogger } from '../../utils/logger.utils';
import { NotFoundError } from '../../utils/errors.utils';
import { DriveItemService } from '../../services/drive/drive-item.service';

export class DELETERequestHandler implements WebDavMethodHandler {
handle = async (req: Request, res: Response) => {
Expand All @@ -16,6 +17,7 @@ export class DELETERequestHandler implements WebDavMethodHandler {
}

await WebDavUtils.deleteOrTrashItem(driveItem);
DriveItemService.instance.forgetFileWrites(resource.url);

res.status(204).send();
};
Expand Down
8 changes: 8 additions & 0 deletions src/webdav/handlers/MOVE.handler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,10 @@
import { webdavLogger } from '../../utils/logger.utils';
import { WebDavUtils } from '../../utils/webdav.utils';
import { WebDavFolderService } from '../../services/webdav/webdav-folder.service';
import { DriveItemService } from '../../services/drive/drive-item.service';

export class MOVERequestHandler implements WebDavMethodHandler {
handle = async (req: Request, res: Response) => {

Check failure on line 13 in src/webdav/handlers/MOVE.handler.ts

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Refactor this function to reduce its Cognitive Complexity from 17 to the 15 allowed.

See more on https://sonarcloud.io/project/issues?id=internxt_cli&issues=AaHrQbP9Gh4STSEzbYOZ&open=AaHrQbP9Gh4STSEzbYOZ&pullRequest=736
const resource = await WebDavUtils.getRequestedResource(req.url);

webdavLogger.info(`[MOVE] Request received for item at ${resource.url}`);
Expand Down Expand Up @@ -44,6 +45,7 @@
isOverwrite = true;
webdavLogger.info(`[MOVE] Destination '${destinationResource.url}' already exists, overwriting it`);
await WebDavUtils.deleteOrTrashItem(destinationDriveItem);
DriveItemService.instance.forgetFileWrites(destinationResource.url);
}

if (destinationResource.path.dir === resource.path.dir) {
Expand Down Expand Up @@ -107,6 +109,12 @@
},
]);

if (originalDriveItem.itemType === 'file') {
DriveItemService.instance.moveFileWrite(resource.url, destinationResource.url);
} else {
DriveItemService.instance.forgetFileWrites(resource.url);

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When originalDriveItem is a folder, forgetFileWrites(resource.url) clears recent-write protection for every file inside it, but moveFileWrite is never called to re-register them at their new destination paths (only the single-file branch above does that). A file PUT just before its containing folder is moved/renamed loses the trust window at the new path, so a GET right after the move can still hit the exact replica-lag 404 this PR is meant to prevent.

I dont know if I am missing something but here is an example: a backup client PUTs /Backups/2026-11-18_tmp/report.pdf (201, remembered for 10s). Within that window it issues MOVE /Backups/2026-11-18_tmp -> /Backups/2026-11-18 to atomically "publish" the finished backup (a common pattern: write into a staging folder, then rename it into place). The handler calls forgetFileWrites('/Backups/2026-11-18_tmp/'), dropping the remembered write for report.pdf, but never calls moveFileWrite for it at the new path. If a client now does GET /Backups/2026-11-18/report.pdf to verify the backup and that read hits a replica that hasn't caught up with the folder rename yet, the API returns 404 and there's no trust window to fall back on, since that whole pattern is gone, so the server returns a 404 for a file that was just written seconds ago

}

const statusCode = isOverwrite ? 204 : 201;
res.status(statusCode).send();
};
Expand Down
54 changes: 40 additions & 14 deletions src/webdav/handlers/PUT.handler.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
import { Request, Response } from 'express';
import { DriveFileService } from '../../services/drive/drive-file.service';
import { DriveItemRepository } from '../../services/database/drive-item/drive-item.repository';
import { DriveItemService } from '../../services/drive/drive-item.service';
import { AuthService } from '../../services/auth.service';
import { WebDavMethodHandler } from '../../types/webdav.types';
import { ConflictError, ErrorUtils } from '../../utils/errors.utils';
import { ConflictError, ErrorUtils, ServiceUnavailableError } from '../../utils/errors.utils';
import { WebDavUtils } from '../../utils/webdav.utils';
import { webdavLogger } from '../../utils/logger.utils';
import { EncryptionVersion } from '@internxt/sdk/dist/drive/storage/types';
Expand All @@ -12,6 +13,7 @@ import { WebDavFolderService } from '../../services/webdav/webdav-folder.service
import { ThumbnailService } from '../../services/thumbnail.service';
import { FormatUtils } from '../../utils/format.utils';
import { UploadUtils } from '../../utils/upload.utils';
import { DriveFileItem } from '../../types/drive.types';

export class PUTRequestHandler implements WebDavMethodHandler {
handle = async (req: Request, res: Response) => {
Expand Down Expand Up @@ -109,24 +111,22 @@ export class PUTRequestHandler implements WebDavMethodHandler {
};

let file;
if (driveFileItem?.itemType === 'file') {
if (isReplacement && driveFileItem) {
try {
file = await DriveFileService.instance.replaceFile(driveFileItem.uuid, filePayload);
} catch (error) {
if (ErrorUtils.isClientVersionUnsupported(error)) throw error;
webdavLogger.warn(
ErrorUtils.withRequestId(
`[PUT] File replace failed for '${resource.url}', falling back to delete and create: ${
error instanceof Error ? error.message : String(error)
}`,
error,
),
);
await WebDavUtils.deleteOrTrashItem(driveFileItem);
file = await DriveFileService.instance.createFile(filePayload);
if (!ErrorUtils.isNotFoundError(error) && !ErrorUtils.isFileNotReplaceable(error)) throw error;
// The lookup came from a lagging replica; the primary says this file is gone, so PUT creates it
webdavLogger.info(`[PUT] '${resource.url}' is no longer replaceable, creating it instead`, {
uuid: driveFileItem.uuid,
});
await DriveItemRepository.instance.delete([driveFileItem.uuid]);
DriveItemService.instance.forgetFileWrite(resource.url, driveFileItem.uuid);
isReplacement = false;
file = await this.createFileOrRetryLater(resource.url, filePayload);
}
} else {
file = await DriveFileService.instance.createFile(filePayload);
file = await this.createFileOrRetryLater(resource.url, filePayload, driveFileItem?.uuid);
}
timings.driveUpload = driveTimer.stop();

Expand All @@ -139,6 +139,7 @@ export class PUTRequestHandler implements WebDavMethodHandler {
updatedAt: new Date(),
},
]);
DriveItemService.instance.rememberFileWrite(resource.url, file);

const thumbnailTimer = CLIUtils.timer();
await ThumbnailService.instance.tryUploadThumbnail({
Expand Down Expand Up @@ -168,4 +169,29 @@ export class PUTRequestHandler implements WebDavMethodHandler {
res.header('ETag', WebDavUtils.getItemETag(file));
res.status(statusCode).send();
};

private readonly createFileOrRetryLater = async (
url: string,
payload: Parameters<DriveFileService['createFile']>[0],
staleUuid?: string,
): Promise<DriveFileItem> => {
try {
return await DriveFileService.instance.createFile(payload);
} catch (error) {
// POST /files only returns 409 when a live file with this name exists
if (!ErrorUtils.isApiConflict(error)) throw error;

webdavLogger.warn(
ErrorUtils.withRequestId(
`[PUT] '${url}' conflicts on the API, asking to retry: ${ErrorUtils.getResponseMessage(error) ?? 'no message'}`,
error,
),
);
if (staleUuid) {
await DriveItemRepository.instance.delete([staleUuid]);
DriveItemService.instance.forgetFileWrite(url, staleUuid);
}
throw new ServiceUnavailableError(`File '${url}' already exists but could not be found yet, retry later`);
}
};
}
69 changes: 69 additions & 0 deletions test/services/drive/drive-file.service.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,28 @@ describe('Drive file Service', () => {
expect(result.folderUuid).to.be.equal(payload.folderUuid);
});

test('when the API returns the size as a string, then created and replaced files carry it as a number', async () => {
const payload: Storage.FileEntryByUuid = {
plainName: 'example',
type: 'txt',
size: 1024,
folderUuid: 'folder_uuid',
fileId: 'fileId_123456',
bucket: 'bucket123',
encryptVersion: EncryptionVersion.Aes03,
};
const apiFile = { uuid: 'example-uuid', size: '1024', createdAt: new Date(), updatedAt: new Date() };
const storageClientMock: Partial<Drive.Storage> = {
createFileEntryByUuid: vi.fn().mockResolvedValue(apiFile),
replaceFile: vi.fn().mockResolvedValue(apiFile),
};
// @ts-expect-error - We only stub the methods we need to test
vi.spyOn(SdkManager.instance, 'getStorage').mockReturnValue(storageClientMock);

expect((await sut.createFile(payload)).size).toBe(1024);
expect((await sut.replaceFile('example-uuid', payload)).size).toBe(1024);
});

test('when file metadata is requested, then it is returned successfully', async () => {
const fakeFileData: DriveFileData = {
uuid: randomUUID(),
Expand Down Expand Up @@ -120,4 +142,51 @@ describe('Drive file Service', () => {

await expect(sut.getFileMetadata(randomUUID())).rejects.toBeInstanceOf(ServiceUnavailableError);
});

describe('findExistentFile', () => {
const existent = (type: string | null) => ({
uuid: randomUUID(),
plainName: 'report',
type,
size: '1',
bucket: 'bucket',
fileId: 'file-id',
folderUuid: 'folder-uuid',
status: 'EXISTS',
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
});

const mockDuplicates = (existentFiles: ReturnType<typeof existent>[]) => {
const storageClientMock: Partial<Storage> = {
checkDuplicatedFiles: vi.fn().mockResolvedValue({ existentFiles }),
};
// @ts-expect-error - We only stub the method we need to test
vi.spyOn(SdkManager.instance, 'getStorage').mockReturnValue(storageClientMock);
};

test('when the only duplicate has another extension, then no file is returned', async () => {
mockDuplicates([existent('txt')]);

await expect(sut.findExistentFile('folder-uuid', { plainName: 'report', type: 'pdf' })).resolves.toBeUndefined();
});

test('when several duplicates are returned, then the one with the same extension is picked', async () => {
const match = existent('pdf');
mockDuplicates([existent('txt'), match]);

const result = await sut.findExistentFile('folder-uuid', { plainName: 'report', type: 'pdf' });

expect(result?.uuid).toBe(match.uuid);
});

test('when the file has no extension, then a duplicate without one matches', async () => {
const match = existent(null);
mockDuplicates([existent('txt'), match]);

const result = await sut.findExistentFile('folder-uuid', { plainName: 'report', type: '' });

expect(result?.uuid).toBe(match.uuid);
});
});
});
Loading
Loading