-serialized storage backend access

-better environmentization
-bug fixes

Author: aczwink

portal/src/FileEventsService.ts

@@ -0,0 +1,31 @@
+/**
+ * OpenDistributedFileStorage
+ * Copyright (C) 2024 Amir Czwink ([email protected])
+ * 
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ * 
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ * */
+
+import { Injectable } from "acfrontend";
+import { Subject } from "acts-util-core";
+
+@Injectable
+export class FileEventsService
+{
+    constructor()
+    {
+        this.onChanged = new Subject();
+    }
+    
+    public onChanged: Subject<void>;
+}

portal/src/file-explorer/DirectoryViewComponent.tsx

@@ -16,21 +16,20 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  * */
 
-import { Anchor, AutoCompleteMultiSelectBox, BootstrapIcon, Component, FormField, Injectable, JSX_CreateElement, JSX_Fragment, LineEdit, ProgressSpinner, RouteParamProperty, Router } from "acfrontend";
+import { Anchor, AutoCompleteMultiSelectBox, BootstrapIcon, Component, FormField, Injectable, JSX_CreateElement, JSX_Fragment, LineEdit, ProgressSpinner, RouteParamProperty } from "acfrontend";
 import { APIService } from "../APIService";
 import { DirectoryContentsDTO } from "../../dist/api";
 import { FilesGridView } from "./FilesGridView";
 import { FilesTableView } from "./FilesTableView";
 
 @Injectable
-export class DirectoryViewComponent extends Component
+export class DirectoryViewComponent extends Component<{ dirPath: string }>
 {
-    constructor(private apiService: APIService, @RouteParamProperty("containerId") private containerId: number, private router: Router)
+    constructor(private apiService: APIService, @RouteParamProperty("containerId") private containerId: number)
     {
         super();
 
         this.containerName = "";
-        this.dirPath = "/";
         this.data = null;
         this.view = "grid";
         this.showSearch = false;
@@ -62,17 +61,6 @@ export class DirectoryViewComponent extends Component
     }
 
     //Private methods
-    private ExtractDirPath()
-    {
-        const dirPath = this.router.state.Get().queryParams.dirPath;
-        if(dirPath !== undefined)
-            this.dirPath = decodeURIComponent(dirPath);
-        else
-            this.dirPath = "/";
-
-        this.LoadData();
-    }
-
     private GetToggleButtonClassName(condition: boolean)
     {
         const className = condition ? "btn-primary btn-active" : "text-primary";
@@ -82,7 +70,7 @@ export class DirectoryViewComponent extends Component
     private async LoadData()
     {
         this.data = null;
-        const response = await this.apiService.containers._any_.files.get(this.containerId, { dirPath: this.dirPath });
+        const response = await this.apiService.containers._any_.files.get(this.containerId, { dirPath: this.input.dirPath });
         if(response.statusCode !== 200)
             throw new Error("TODO: implement me");
         this.data = response.data;
@@ -94,8 +82,8 @@ export class DirectoryViewComponent extends Component
             return <ProgressSpinner />;
 
         if(this.view === "grid")
-            return <FilesGridView containerId={this.containerId} contents={this.data} parentPath={this.dirPath} />;
-        return <FilesTableView containerId={this.containerId} contents={this.data} parentPath={this.dirPath} />;
+            return <FilesGridView containerId={this.containerId} contents={this.data} parentPath={this.input.dirPath} />;
+        return <FilesTableView containerId={this.containerId} contents={this.data} parentPath={this.input.dirPath} />;
     }
 
     private RenderNav()
@@ -105,9 +93,9 @@ export class DirectoryViewComponent extends Component
             return "/" + dirs.slice(0, idx + 1).join("/");
         }
 
-        if(this.dirPath === "/")
+        if(this.input.dirPath === "/")
             return <li className="breadcrumb-item active">{this.containerName}</li>;
-        const dirs = this.dirPath.substring(1).split("/");
+        const dirs = this.input.dirPath.substring(1).split("/");
         const last = dirs.pop();
         return <>
             <li className="breadcrumb-item"><Anchor route={"/" + this.containerId}>{this.containerName}</Anchor></li>
@@ -157,8 +145,12 @@ export class DirectoryViewComponent extends Component
         const response = await this.apiService.containers.get();
         this.containerName = response.data.find(x => x.id === this.containerId)!.name;
 
-        this.ExtractDirPath();
-        this.router.state.Subscribe(this.ExtractDirPath.bind(this));
+        this.LoadData();
+    }
+
+    override OnInputChanged(): void
+    {
+        this.LoadData();
     }
 
     private async OnLoadTags(searchText: string)
@@ -176,7 +168,7 @@ export class DirectoryViewComponent extends Component
 
         this.data = null;
         const response = await this.apiService.containers._any_.search.get(this.containerId, {
-            dirPath: this.dirPath,
+            dirPath: this.input.dirPath,
             nameFilter: this.searchFilterName,
             mediaTypeFilter: this.searchFilterMediaType,
             requiredTags: this.searchTags.join(",")
@@ -197,7 +189,6 @@ export class DirectoryViewComponent extends Component
 
     //State
     private containerName: string;
-    private dirPath: string;
     private data: DirectoryContentsDTO | null;
     private view: "grid" | "list";
     private showSearch: boolean;

portal/src/file-explorer/EditFileAttributesComponent.tsx

@@ -19,6 +19,7 @@
 import { AutoCompleteTextLineEdit, BootstrapIcon, FormField, JSX_CreateElement, LineEdit, PushButton, Router, Use, UseAPI, UseDeferredAPI, UseRouteParameter, UseState } from "acfrontend";
 import { APIService } from "../APIService";
 import { FileMetaDataDTO } from "../../dist/api";
+import { FileEventsService } from "../FileEventsService";
 
 async function LoadTags(containerId: number, searchText: string)
 {
@@ -54,7 +55,10 @@ function FileEditor(input: { containerId: number; fileId: number; fileData: File
             filePath: state.filePath,
             tags: state.tags
         }),
-        () => Use(Router).RouteTo("/" + input.containerId + "/" + input.fileId)
+        () => {
+            Use(FileEventsService).onChanged.Next();
+            Use(Router).RouteTo("/" + input.containerId + "/" + input.fileId);
+        }
     );
     if(apiState.started)
         return apiState.fallback;

portal/src/file-explorer/FileExplorerComponent.tsx

@@ -31,6 +31,7 @@ export class FileExplorerComponent extends Component
         super();
 
         this.loading = false;
+        this.dirPath = "/";
     }
 
     protected Render(): RenderValue
@@ -39,18 +40,30 @@ export class FileExplorerComponent extends Component
             return <ProgressSpinner />;
 
         if(this.apiService.readOnly)
-            return <DirectoryViewComponent />;
+            return <DirectoryViewComponent dirPath={this.dirPath} />;
         return <div ondragenter={this.OnDragEnter.bind(this)} ondragleave={this.OnDragLeave.bind(this)} ondragover={this.OnDragOver.bind(this)} ondrop={this.OnDrop.bind(this)} style="min-height: 95vh">
-            <DirectoryViewComponent />
+            <DirectoryViewComponent dirPath={this.dirPath} />
         </div>;
     }
 
     //Private methods
+    private ExtractDirPath()
+    {
+        const dirPath = this.router.state.Get().queryParams.dirPath;
+        if(dirPath !== undefined)
+            this.dirPath = decodeURIComponent(dirPath);
+        else
+            this.dirPath = "/";
+    }
+
     private async UploadFile(file: File)
     {
+        console.log("Uploading file", file.name);
         const response = await this.apiService.containers._any_.files.post(this.containerId, {
+            parentPath: this.dirPath,
             file
         });
+        console.log("Finished", file.name, "result: ", response);
         switch(response.statusCode)
         {
             case 200:
@@ -150,6 +163,13 @@ export class FileExplorerComponent extends Component
         await this.UploadFiles(files);
     }
 
+    override OnInitiated(): void
+    {
+        this.ExtractDirPath();
+        this.router.state.Subscribe(this.ExtractDirPath.bind(this));
+    }
+
     //State
+    private dirPath: string;
     private loading: boolean;
 }

portal/src/file-explorer/ViewFileComponent.tsx

@@ -19,6 +19,8 @@
 import { Injectable, Component, RouteParamProperty, ProgressSpinner, JSX_CreateElement, JSX_Fragment, BootstrapIcon, Anchor, FileDownloadService, NavItem, RouterComponent, InfoMessageManager } from "acfrontend";
 import { APIService } from "../APIService";
 import { FileMetaDataDTO } from "../../dist/api";
+import { FileEventsService } from "../FileEventsService";
+import { Subscription } from "acts-util-core";
 
 let dragCounter = 0;
 
@@ -28,6 +30,7 @@ export class ViewFileComponent extends Component
     constructor(private apiService: APIService, private fileDownloadService: FileDownloadService, private infoMessageManager: InfoMessageManager,
         @RouteParamProperty("containerId") private containerId: number,
         @RouteParamProperty("fileId") private fileId: number,
+        private fileEventsService: FileEventsService
     )
     {
         super();
@@ -225,9 +228,19 @@ export class ViewFileComponent extends Component
             throw new Error("TODO: implement me");
         this.containerName = response.data.name;
         this.LoadData();
+
+        this.subscription = this.fileEventsService.onChanged.Subscribe({
+            next: () => this.LoadData()
+        });
+    }
+
+    override OnUnmounted(): void
+    {
+        this.subscription?.Unsubscribe();
     }
 
     //State
     private data: FileMetaDataDTO | null;
     private containerName: string;
+    private subscription?: Subscription;
 }

service/src/api/containers.ts

@@ -133,10 +133,11 @@ class _api3_
     @Security(OIDC_API_SCHEME, [SCOPE_FILES_WRITE])
     public async Create(
         @Path containerId: number,
+        @FormField parentPath: string,
         @FormField file: UploadedFile
     )
     {
-        const result = await this.fileUploadService.Upload(containerId, file.originalName, file.mediaType, file.buffer);
+        const result = await this.fileUploadService.Upload(containerId, parentPath, file.originalName, file.mediaType, file.buffer);
         if(result === "error_file_exists")
             return Conflict("file exists already");
         return result;

service/src/data-access/FilesController.ts

@@ -45,12 +45,12 @@ export class FilesController
     }
 
     //Public methods
-    public async AddFile(containerId: number, originalName: string, mediaType: string)
+    public async AddFile(containerId: number, filePath: string, mediaType: string)
     {
         const conn = await this.dbConnMgr.CreateAnyConnectionQueryExecutor();
         const result = await conn.InsertRow("files", {
             containerId,
-            filePath: "/" + originalName,
+            filePath,
             mediaType,
         });
 

service/src/env.ts

@@ -22,6 +22,7 @@ export const CONFIG_DB = {
     password: process.env.ODFS_DBPW!,
 };
 
+export const CONFIG_MAX_NUMBER_OF_CACHED_BLOCKS = parseInt(process.env.ODFS_MAX_CACHE_SIZE!); //in units of 100 MiB
 export const CONFIG_OIDP_ENDPOINT = process.env.ODFS_OIDP_ENDPOINT!;
 export const CONFIG_ORIGIN = process.env.ODFS_ORIGIN!;
 export const CONFIG_PORT = process.env.ODFS_PORT;

service/src/services/FFProbeService.ts

@@ -93,7 +93,15 @@ export class FFProbeService
         switch(stream.codec_type)
         {
             case "audio":
-                return stream.codec_name === "mp3";
+            {
+                switch(stream.codec_name)
+                {
+                    case "aac":
+                    case "mp3":
+                        return true;
+                }
+                return false;
+            }
             case "video":
                 return stream.codec_name === "h264";
         }

service/src/services/FileUploadService.ts

@@ -16,6 +16,7 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  * */
 import crypto from "crypto";
+import path from "path";
 import { Readable } from 'stream';
 import { Injectable } from "acts-util-node";
 import { BlobsController } from "../data-access/BlobsController";
@@ -34,13 +35,14 @@ export class FileUploadService
     }
 
     //Public methods
-    public async Upload(containerId: number, originalName: string, mediaType: string, buffer: Buffer)
+    public async Upload(containerId: number, parentPath: string, originalName: string, mediaType: string, buffer: Buffer)
     {
         const blobId = await this.ProcessBlob(Readable.from(buffer));
         let fileId;
         try
         {
-            fileId = await this.filesController.AddFile(containerId, originalName, mediaType);
+            const filePath = path.join(parentPath, originalName);
+            fileId = await this.filesController.AddFile(containerId, filePath, mediaType);
         }
         catch(e: any)
         {
@@ -148,28 +150,21 @@ export class FileUploadService
     {
         const blockId = await this.blobsController.FindBlobBlock(blobBlock.byteLength, sha256sum);
         if(blockId !== undefined)
-        {
-            console.log("found");
             return blockId;
-        }
 
         let newBlockId;
         try
         {
             newBlockId = await this.blobsController.AddBlobBlock(blobBlock.byteLength, sha256sum);
-            console.log("added blob block", newBlockId, blobBlock.byteLength, sha256sum);
         }
         catch(e: any)
         {
-            console.log("error", e);
             if(e?.code === "ER_DUP_ENTRY")
                 return await this.ProcessBlobBlockHashed(blobBlock, sha256sum);
             throw e;
         }
-        console.log("adding", newBlockId);
         const storageBlock = await this.storageBackendsManager.StoreBlobBlock(blobBlock);
         await this.blobsController.AddBlobBlockStorage(newBlockId, storageBlock.id, storageBlock.offset);
-        console.log("added", newBlockId, storageBlock.id, storageBlock.offset);
 
         return newBlockId;
     }

service/src/services/StorageBackendsManager.ts

@@ -25,6 +25,7 @@ import { Dictionary, ObjectExtensions } from "acts-util-core";
 import { StorageBlocksController } from "../data-access/StorageBlocksController";
 import { CONST_STORAGEBLOCKS_MAX_REPLICATION, StorageTier } from "../constants";
 import { WebDAVBackend } from "../storage-backends/WebDAVBackend";
+import { LockedStorageBackend } from "../storage-backends/LockedStorageBackend";
 
 export interface StorageBackendCreationData
 {
@@ -100,6 +101,11 @@ export class StorageBackendsManager
 
     //Private methods
     private CreateBackendInstance(config: StorageBackendConfig): StorageBackend
+    {
+        return new LockedStorageBackend(this.CreateRealBackendInstance(config));
+    }
+
+    private CreateRealBackendInstance(config: StorageBackendConfig): StorageBackend
     {
         switch(config.type)
         {