Skip to content

Commit 2ae215b

Browse files
committed
-implemented soft file deletion
-implemented garbage collection -resolved #25 -resolved #26 -resolved #24 -bug fixes
1 parent e1721bb commit 2ae215b

File tree

69 files changed

+2145
-227
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

69 files changed

+2145
-227
lines changed

infrastructure/db.sql

Lines changed: 39 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -108,10 +108,12 @@ DROP TABLE IF EXISTS `blobs_versions`;
108108
/*!40101 SET character_set_client = utf8 */;
109109
CREATE TABLE `blobs_versions` (
110110
`blobId` int(10) unsigned NOT NULL,
111+
`versionBlobId` int(10) unsigned NOT NULL,
111112
`title` varchar(100) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
112-
PRIMARY KEY (`blobId`,`title`),
113-
KEY `files_versions_blobId` (`blobId`),
114-
CONSTRAINT `files_versions_blobId` FOREIGN KEY (`blobId`) REFERENCES `blobs` (`id`)
113+
PRIMARY KEY (`blobId`,`versionBlobId`),
114+
KEY `files_versions_versionBlobId` (`versionBlobId`),
115+
CONSTRAINT `files_versions_blobId` FOREIGN KEY (`blobId`) REFERENCES `blobs` (`id`),
116+
CONSTRAINT `files_versions_versionBlobId` FOREIGN KEY (`versionBlobId`) REFERENCES `blobs` (`id`)
115117
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
116118
/*!40101 SET character_set_client = @saved_cs_client */;
117119

@@ -162,6 +164,39 @@ CREATE TABLE `files` (
162164
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
163165
/*!40101 SET character_set_client = @saved_cs_client */;
164166

167+
--
168+
-- Table structure for table `files_deleted`
169+
--
170+
171+
DROP TABLE IF EXISTS `files_deleted`;
172+
/*!40101 SET @saved_cs_client = @@character_set_client */;
173+
/*!40101 SET character_set_client = utf8 */;
174+
CREATE TABLE `files_deleted` (
175+
`fileId` int(10) unsigned NOT NULL,
176+
`deletionTime` datetime NOT NULL,
177+
PRIMARY KEY (`fileId`),
178+
CONSTRAINT `files_deleted_fileId` FOREIGN KEY (`fileId`) REFERENCES `files` (`id`)
179+
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
180+
/*!40101 SET character_set_client = @saved_cs_client */;
181+
182+
--
183+
-- Table structure for table `files_locations`
184+
--
185+
186+
DROP TABLE IF EXISTS `files_locations`;
187+
/*!40101 SET @saved_cs_client = @@character_set_client */;
188+
/*!40101 SET character_set_client = utf8 */;
189+
CREATE TABLE `files_locations` (
190+
`fileId` int(10) unsigned NOT NULL,
191+
`lat` float NOT NULL,
192+
`lon` float NOT NULL,
193+
`countryCode` char(2) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
194+
`osmId` varchar(50) CHARACTER SET ascii COLLATE ascii_bin DEFAULT NULL,
195+
PRIMARY KEY (`fileId`),
196+
CONSTRAINT `files_locations_fileId` FOREIGN KEY (`fileId`) REFERENCES `files` (`id`)
197+
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
198+
/*!40101 SET character_set_client = @saved_cs_client */;
199+
165200
--
166201
-- Table structure for table `files_revisions`
167202
--
@@ -287,4 +322,4 @@ CREATE TABLE `tags` (
287322
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
288323
/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
289324

290-
-- Dump completed on 2024-12-15 21:19:33
325+
-- Dump completed on 2024-12-22 22:16:04

portal/package.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
},
2222
"dependencies": {
2323
"acfrontend": "*",
24-
"acts-util-core": "*"
24+
"acts-util-core": "*",
25+
"country-iso-2-to-3": "^1.1.0"
2526
}
2627
}

portal/src/RootComponent.tsx

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@
1717
* */
1818
import { BootstrapIcon, Component, Injectable, JSX_CreateElement, JSX_Fragment, Navigation, NavItem, OAuth2Service, OAuth2TokenManager, Router, RouterComponent } from "acfrontend";
1919
import { CONFIG_OIDC } from "./config";
20-
import { APIService } from "./APIService";
2120
import { SCOPE_FILES_WRITE } from "./definitions";
21+
import { APIService } from "./services/APIService";
2222

2323
@Injectable
2424
export class RootComponent extends Component
@@ -59,6 +59,7 @@ export class RootComponent extends Component
5959
{
6060
if(this.router.state.Get().ToUrl().path.startsWith("/settings"))
6161
return <NavItem route="/"><BootstrapIcon>house</BootstrapIcon></NavItem>;
62+
6263
return this.RenderEditCheck();
6364
}
6465

portal/src/SettingsComponent.tsx

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ export function SettingsComponent()
2626
<ul className="nav nav-pills flex-column">
2727
<NavItem route={"/settings/containers"}><BootstrapIcon>eyeglasses</BootstrapIcon> Containers</NavItem>
2828
<NavItem route={"/settings/storagebackends"}><BootstrapIcon>eyeglasses</BootstrapIcon> Storage backends</NavItem>
29+
<NavItem route={"/settings/reporting"}><BootstrapIcon>file-text</BootstrapIcon> Reporting</NavItem>
2930
</ul>
3031
</div>
3132
<div className="col">

portal/src/containers/CreateContainersComponent.tsx

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,9 @@
1717
* */
1818

1919
import { BootstrapIcon, FormField, JSX_CreateElement, JSX_Fragment, LineEdit, PushButton, Router, Use, UseDeferredAPI, UseState } from "acfrontend";
20-
import { APIService } from "../APIService";
2120
import { APIResponse } from "acfrontend/dist/RenderHelpers";
2221
import { ContainerProperties } from "../../dist/api";
22+
import { APIService } from "../services/APIService";
2323

2424
function ContainerFormComponent(input: { saveAPI: (data: ContainerProperties) => Promise<APIResponse<void>> })
2525
{

portal/src/containers/ListContainersComponent.tsx

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@
1717
* */
1818

1919
import { BootstrapIcon, JSX_CreateElement, JSX_Fragment, RouterButton, Use, UseAPI } from "acfrontend";
20-
import { APIService } from "../APIService";
2120
import { ContainerProperties } from "../../dist/api";
21+
import { APIService } from "../services/APIService";
2222

2323
function ContainersList(input: { containers: ContainerProperties[] })
2424
{

portal/src/file-explorer/ContainerSelectionComponent.tsx

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@
1717
* */
1818

1919
import { UseAPI, Use, JSX_CreateElement, BootstrapIcon, Anchor } from "acfrontend";
20-
import { APIService } from "../APIService";
2120
import { Container } from "../../dist/api";
21+
import { APIService } from "../services/APIService";
2222

2323
function ContainersList(input: { containers: Container[] })
2424
{

portal/src/file-explorer/CreateFileVersionComponent.tsx

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,9 @@
1717
* */
1818

1919
import { BootstrapIcon, FormField, JSX_CreateElement, PushButton, Router, Select, Use, UseAPI, UseDeferredAPI, UseRouteParameter, UseState } from "acfrontend";
20-
import { APIService } from "../APIService";
2120
import { FileMetaDataDTO, StreamingVersionType } from "../../dist/api";
2221
import { Of } from "acts-util-core";
22+
import { APIService } from "../services/APIService";
2323

2424
function CreateFileVersionForm(input: { containerId: number; fileId: number; metadata: FileMetaDataDTO })
2525
{

portal/src/file-explorer/DirectoryViewComponent.tsx

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,11 @@
1616
* along with this program. If not, see <http://www.gnu.org/licenses/>.
1717
* */
1818

19-
import { Anchor, AutoCompleteMultiSelectBox, BootstrapIcon, Component, FormField, Injectable, JSX_CreateElement, JSX_Fragment, LineEdit, ProgressSpinner, RouteParamProperty } from "acfrontend";
20-
import { APIService } from "../APIService";
19+
import { Anchor, AutoCompleteMultiSelectBox, BootstrapIcon, Component, FormField, Injectable, JSX_CreateElement, JSX_Fragment, LineEdit, ProgressSpinner, RouteParamProperty, RouterButton } from "acfrontend";
2120
import { DirectoryContentsDTO } from "../../dist/api";
2221
import { FilesGridView } from "./FilesGridView";
2322
import { FilesTableView } from "./FilesTableView";
23+
import { APIService } from "../services/APIService";
2424

2525
@Injectable
2626
export class DirectoryViewComponent extends Component<{ dirPath: string }>
@@ -51,6 +51,7 @@ export class DirectoryViewComponent extends Component<{ dirPath: string }>
5151
<br />
5252
<button type="button" className={this.GetToggleButtonClassName(this.view === "grid")} onclick={() => this.view = "grid"}><BootstrapIcon>grid</BootstrapIcon></button>
5353
<button type="button" className={this.GetToggleButtonClassName(this.view === "list")} onclick={() => this.view = "list"}><BootstrapIcon>view-list</BootstrapIcon></button>
54+
<Anchor route={"/" + this.containerId + "/maps"}><BootstrapIcon>globe-europe-africa</BootstrapIcon></Anchor>
5455
</li>
5556
</ol>
5657
</nav>
Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
/**
2+
* OpenDistributedFileStorage
3+
* Copyright (C) 2024 Amir Czwink ([email protected])
4+
*
5+
* This program is free software: you can redistribute it and/or modify
6+
* it under the terms of the GNU Affero General Public License as published by
7+
* the Free Software Foundation, either version 3 of the License, or
8+
* (at your option) any later version.
9+
*
10+
* This program is distributed in the hope that it will be useful,
11+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
12+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13+
* GNU Affero General Public License for more details.
14+
*
15+
* You should have received a copy of the GNU Affero General Public License
16+
* along with this program. If not, see <http://www.gnu.org/licenses/>.
17+
* */
18+
import { RootInjector, FileDownloadService, PopupManager, ProgressSpinner, JSX_CreateElement, Component } from "acfrontend";
19+
import { Property } from "acts-util-core";
20+
import { ResponseData } from "../../dist/api";
21+
22+
class DownloadModalPopup extends Component<{ progress: Property<ProgressEvent | null>; startTime: number; }>
23+
{
24+
protected override Render(): RenderValue
25+
{
26+
return <div className="modal-dialog">
27+
<div className="modal-content">
28+
<div className="modal-body">
29+
<p>Downloading file. Standby...</p>
30+
{this.RenderStatus()}
31+
</div>
32+
</div>
33+
</div>;
34+
}
35+
36+
//Private methods
37+
private RenderStatus()
38+
{
39+
const v = this.input.progress.Get();
40+
if(v === null)
41+
return <ProgressSpinner />;
42+
43+
const dt = (Date.now() - this.input.startTime) / 1000;
44+
const speed = v.loaded / dt;
45+
if(v.lengthComputable)
46+
{
47+
const percent = Math.round(v.loaded / v.total * 100);
48+
return <fragment>
49+
<div className="progress">
50+
<div className="progress-bar" style={"width: " + percent + "%"}>{percent}%</div>
51+
</div>
52+
<br />
53+
Downloaded: {v.loaded.FormatBinaryPrefixed("B")} of {v.total.FormatBinaryPrefixed("B")} ({speed.FormatBinaryPrefixed("B")}/s)
54+
</fragment>;
55+
}
56+
return "Downloaded: " + v.loaded.FormatBinaryPrefixed("B") + " (" + speed.FormatBinaryPrefixed("B") + "/s)";
57+
}
58+
59+
//Event handlers
60+
override OnInitiated(): void
61+
{
62+
this.input.progress.Subscribe(this.Update.bind(this));
63+
}
64+
}
65+
66+
export async function DownloadFileUsingProgressPopup(fileName: string, initRequest: (progressTracker: (event: ProgressEvent) => void) => Promise<ResponseData<number, number, Blob>>)
67+
{
68+
const prop = new Property<ProgressEvent | null>(null);
69+
const ref = RootInjector.Resolve(PopupManager).OpenModal(<DownloadModalPopup progress={prop} startTime={Date.now()} />, { className: "fade show d-block" });
70+
71+
const response = await initRequest(event => prop.Set(event));
72+
73+
ref.Close();
74+
if("data" in response)
75+
RootInjector.Resolve(FileDownloadService).DownloadBlobAsFile(response.data, fileName);
76+
else
77+
throw new Error("TODO: implement me!");
78+
return response;
79+
}

portal/src/file-explorer/EditFileAttributesComponent.tsx

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,11 @@
1717
* */
1818

1919
import { AutoCompleteTextLineEdit, BootstrapIcon, FormField, JSX_CreateElement, LineEdit, PushButton, Router, Use, UseAPI, UseDeferredAPI, UseRouteParameter, UseState } from "acfrontend";
20-
import { APIService } from "../APIService";
2120
import { FileMetaDataDTO } from "../../dist/api";
2221
import { FileEventsService } from "../FileEventsService";
22+
import { APIService } from "../services/APIService";
23+
import { GeoLocationSelector } from "../geolocation/GeoLocationSelector";
24+
import { GeoLocationMap } from "../geolocation/GeoLocationMap";
2325

2426
async function LoadTags(containerId: number, searchText: string)
2527
{
@@ -43,15 +45,30 @@ function FileEditor(input: { containerId: number; fileId: number; fileData: File
4345
state.tags[index] = newValue;
4446
state.tags = [...state.tags]; //inform view about change
4547
}
48+
async function onLocationChanged(newValue: string)
49+
{
50+
const response = await Use(APIService).geocoding._any_.get(newValue);
51+
if(response.statusCode !== 200)
52+
throw new Error("TODO implement me");
53+
const location = response.data;
54+
state.locationPos = {
55+
lat: parseFloat(location.latitude),
56+
lon: parseFloat(location.longitude)
57+
};
58+
state.osmLocationId = newValue;
59+
}
4660

4761
const state = UseState({
4862
filePath: input.fileData.filePath,
63+
osmLocationId: input.fileData.location?.osmId ?? null,
64+
locationPos: (input.fileData.location === undefined) ? null : { lat: input.fileData.location.lat, lon: input.fileData.location.lon },
4965
tags: input.fileData.tags
5066
});
5167
const isValid = state.tags.find(x => x.trim().length === 0) === undefined;
5268

5369
const apiState = UseDeferredAPI(
5470
() => Use(APIService).files._any_.put(input.fileId, {
71+
osmLocationId: state.osmLocationId,
5572
filePath: state.filePath,
5673
tags: state.tags
5774
}),
@@ -67,6 +84,9 @@ function FileEditor(input: { containerId: number; fileId: number; fileData: File
6784
<FormField title="File path">
6885
<LineEdit link={state.links.filePath} />
6986
</FormField>
87+
<h4>Location</h4>
88+
<GeoLocationSelector locationId={state.osmLocationId} onValueChanged={onLocationChanged} />
89+
{(state.locationPos === null) ? null : <GeoLocationMap points={[state.locationPos]} /> }
7090
<h4>Tags</h4>
7191
{state.tags.map( (x, i) => <div className="row">
7292
<div className="col">

portal/src/file-explorer/FileAccessesComponent.tsx

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@
1717
* */
1818

1919
import { UseRouteParameter, UseAPI, Use, JSX_CreateElement, JSX_Fragment, BootstrapIcon } from "acfrontend";
20-
import { APIService } from "../APIService";
2120
import { AccessStatistics, StorageTier } from "../../dist/api";
21+
import { APIService } from "../services/APIService";
2222

2323
function RenderStorageTier(storageTier: StorageTier)
2424
{

portal/src/file-explorer/FileExplorerComponent.tsx

Lines changed: 13 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -16,16 +16,17 @@
1616
* along with this program. If not, see <http://www.gnu.org/licenses/>.
1717
* */
1818

19-
import { Component, InfoMessageManager, Injectable, JSX_CreateElement, ProgressSpinner, RouteParamProperty, Router } from "acfrontend";
20-
import { APIService } from "../APIService";
19+
import { Component, InfoMessageManager, Injectable, JSX_CreateElement, PopupManager, ProgressSpinner, RouteParamProperty, Router } from "acfrontend";
2120
import { DirectoryViewComponent } from "./DirectoryViewComponent";
21+
import { APIService } from "../services/APIService";
22+
import { UploadFileModal } from "./UploadFileModal";
2223

2324
let dragCounter = 0;
2425

2526
@Injectable
2627
export class FileExplorerComponent extends Component
2728
{
28-
constructor(private apiService: APIService, private infoMessageManager: InfoMessageManager, private router: Router,
29+
constructor(private apiService: APIService, private infoMessageManager: InfoMessageManager, private router: Router, private popupManager: PopupManager,
2930
@RouteParamProperty("containerId") private containerId: number)
3031
{
3132
super();
@@ -56,53 +57,23 @@ export class FileExplorerComponent extends Component
5657
this.dirPath = "/";
5758
}
5859

59-
private async UploadFile(file: File)
60-
{
61-
console.log("Uploading file", file.name);
62-
const response = await this.apiService.containers._any_.files.post(this.containerId, {
63-
parentPath: this.dirPath,
64-
file
65-
});
66-
console.log("Finished", file.name, "result: ", response);
67-
switch(response.statusCode)
68-
{
69-
case 204:
70-
return true;
71-
case 409:
72-
this.infoMessageManager.ShowMessage(<p>{file.name} was not uploaded because it exists already!</p>, { type: "warning" });
73-
break;
74-
default:
75-
this.infoMessageManager.ShowMessage(<p>Failed uploading file {file.name}</p>, { type: "danger" });
76-
}
77-
78-
return false;
79-
}
80-
8160
private async UploadFiles(files: File[])
8261
{
8362
this.loading = true;
8463

85-
if(files.length === 1)
64+
const context = this;
65+
function OnFinish(success: boolean)
8666
{
87-
const result = await this.UploadFile(files[0]);
88-
if(result)
89-
this.infoMessageManager.ShowMessage(<p>File uploaded successfully. You will see it soon in the explorer...</p>, { type: "success" });
90-
}
91-
else
92-
{
93-
let okCount = 0;
94-
for (const file of files)
67+
if(success)
9568
{
96-
const result = await this.UploadFile(file);
97-
if(result)
98-
okCount++;
69+
if(files.length === 1)
70+
context.infoMessageManager.ShowMessage(<p>File uploaded successfully. You will see it soon in the explorer...</p>, { type: "success" });
71+
else
72+
context.infoMessageManager.ShowMessage(<p>{files.length} files uploaded successfully. You will see them soon in the explorer...</p>, { type: "success" });
9973
}
100-
101-
if(okCount === files.length)
102-
this.infoMessageManager.ShowMessage(<p>{okCount} files uploaded successfully. You will see them soon in the explorer...</p>, { type: "success" });
103-
else
104-
this.infoMessageManager.ShowMessage(<p>{files.length - okCount} files of {files.length} could not be uploaded successfully.</p>, { type: "danger" });
10574
}
75+
this.popupManager.OpenModal(<UploadFileModal context={{ type: "newfile", containerId: this.containerId, parentPath: this.dirPath }} files={files} onFinish={OnFinish} />, { className: "fade show d-block" });
76+
//this.infoMessageManager.ShowMessage(<p>{files.length - okCount} files of {files.length} could not be uploaded successfully.</p>, { type: "danger" });
10677

10778
this.loading = false;
10879
}

0 commit comments

Comments
 (0)