-performance optimization

-bug fixes

Author: aczwink

service/package.json

@@ -6,6 +6,7 @@
   "scripts": {
     "build": "npx acts-util-apilib && npx webpack",
     "build-docker-image": "docker build -t odfs .",
+    "run-dev-rmq": "docker run -it --rm --name rabbitmq -p 5672:5672 -p 15672:15672 rabbitmq:4.0-management",
     "run-dev-server": "nodemon --exec \"npx acts-util-apilib; tsc; node dist/src/main.js\""
   },
   "keywords": [],

service/src/data-access/FilesController.ts

@@ -83,6 +83,7 @@ export class FilesController
 
         await conn.value.DeleteRows("files_locations", "fileId = ?", fileId);
         await conn.value.DeleteRows("files_revisions", "fileId = ?", fileId);
+        await conn.value.DeleteRows("files_tags", "fileId = ?", fileId);
 
         await conn.value.StartTransaction();
         await conn.value.DeleteRows("files_deleted", "fileId = ?", fileId);

service/src/services/StorageBlocksCache.ts

@@ -1,6 +1,6 @@
 /**
  * OpenDistributedFileStorage
- * Copyright (C) 2024 Amir Czwink ([email protected])
+ * Copyright (C) 2024-2025 Amir Czwink ([email protected])
  * 
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as published by
@@ -38,6 +38,12 @@ export class StorageBlocksCache
         this.decryptedStorageBlocks = {};
     }
 
+    //Properties
+    public get cachedBlocksCount()
+    {
+        return ObjectExtensions.OwnKeys(this.decryptedStorageBlocks).Count();
+    }
+
     //Public methods
     public AddToCache(storageBlockId: number, decrypted: Buffer)
     {
@@ -64,7 +70,7 @@ export class StorageBlocksCache
     //Private methods
     private PruneCacheIfFull()
     {
-        const pruneCount = ObjectExtensions.OwnKeys(this.decryptedStorageBlocks).Count() - CONFIG_MAX_NUMBER_OF_CACHED_BLOCKS;
+        const pruneCount = this.cachedBlocksCount - CONFIG_MAX_NUMBER_OF_CACHED_BLOCKS;
         if(pruneCount <= 0)
             return;
         const toPrune = ObjectExtensions.Entries(this.decryptedStorageBlocks).OrderBy(kv => kv.value!.lastAccess).Map(kv => kv.key).Take(pruneCount);

service/src/services/StorageBlocksManager.ts

@@ -24,6 +24,7 @@ import { StorageBackendsManager } from "./StorageBackendsManager";
 import { StorageBlocksCache } from "./StorageBlocksCache";
 import { JobOrchestrationService } from "./JobOrchestrationService";
 import { BlobsController } from "../data-access/BlobsController";
+import { NumberDictionary } from "acts-util-core";
 
 @Injectable
 export class StorageBlocksManager
@@ -33,6 +34,7 @@ export class StorageBlocksManager
         private jobOrchestrationService: JobOrchestrationService, private blobsController: BlobsController
     )
     {
+        this.inFlightRequests = {};
         this.storageBlockAcquirementLock = new Lock;
     }
 
@@ -60,12 +62,17 @@ export class StorageBlocksManager
         if(cached !== undefined)
             return cached;
 
-        const read = await this.DownloadEncryptedStorageBlock(storageBlockId);
-        const encryptionInfo = await this.storageBlocksController.QueryEncryptionInfo(storageBlockId);
-        const decrypted = await this.storageEncryptionManager.Decrypt(this.GetPartitionNumber(storageBlockId), read, encryptionInfo!.iv, encryptionInfo!.authTag);
+        const inFlightRequest = this.inFlightRequests[storageBlockId];
+        if(inFlightRequest !== undefined)
+            return inFlightRequest;
 
-        this.storageBlocksCache.AddToCache(storageBlockId, decrypted);
-        return decrypted;
+        const promise = this.DownloadAndDecryptStorageBlock(storageBlockId);
+        this.inFlightRequests[storageBlockId] = promise;
+
+        const result = await promise;
+        this.inFlightRequests[storageBlockId] = undefined;
+
+        return result;
     }
 
     public async FreeUnreferencedStorageBlock(storageBlockId: number)
@@ -169,6 +176,16 @@ export class StorageBlocksManager
         });
     }
 
+    private async DownloadAndDecryptStorageBlock(storageBlockId: number)
+    {
+        const read = await this.DownloadEncryptedStorageBlock(storageBlockId);
+        const encryptionInfo = await this.storageBlocksController.QueryEncryptionInfo(storageBlockId);
+        const decrypted = await this.storageEncryptionManager.Decrypt(this.GetPartitionNumber(storageBlockId), read, encryptionInfo!.iv, encryptionInfo!.authTag);
+
+        this.storageBlocksCache.AddToCache(storageBlockId, decrypted);
+        return decrypted;
+    }
+
     private async DownloadEncryptedStorageBlock(storageBlockId: number)
     {
         const backend = await this.storageBackendsManager.FindFastestBackendForReading(storageBlockId);
@@ -264,5 +281,6 @@ export class StorageBlocksManager
     }
 
     //State
+    private inFlightRequests: NumberDictionary<Promise<Buffer>>;
     private storageBlockAcquirementLock: Lock;
 }