feat: add Confluence integration with configurable parameters to admin-frontend (#19)

* feat: add Confluence integration with configurable parameters for document loading

* chore: update submodules to latest main

* feat: enable MinIO feature in Helm chart configuration and remove default max pages limit in document upload

* feat: add pytest configuration and update testing setup across multiple components

* Update frontend/libs/i18n/admin/en.json

Co-authored-by: Copilot <[email protected]>

* feat: enhance Confluence upload form with additional input fields and labels

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: Copilot <[email protected]>

Author: 3 people

.vscode/launch.json

@@ -1,6 +1,22 @@
 {
     "version": "0.2.0",
     "configurations": [
+        {
+            "name": "Debug Pytest (current file)",
+            "type": "python",
+            "request": "launch",
+            "python": "${command:python.interpreterPath}",
+            // run pytest as a module
+            "module": "pytest",
+            "args": [
+              "--maxfail=1",
+              "--disable-warnings",
+              "-q",
+              "${file}"
+            ],
+            "console": "integratedTerminal",
+            "justMyCode": false,
+        },
         {
             "name": "rag_backend",
             "type": "python",

.vscode/settings.json

@@ -6,11 +6,22 @@
         "./rag-core-library/rag-core-api/src",
         "./rag-core-library/rag-core-lib/src",
         "./rag-core-library/extractor-api-lib/src",
+        "./admin-backend",
+        "./rag-backend",
+        "./document-extractor"
     ],
     "[yaml]": {
         "editor.tabSize": 2,
         "editor.insertSpaces": true,
         "editor.formatOnType": true,
         "editor.autoIndent": "advanced"
     },
+    "python.testing.pytestEnabled": true,
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestArgs": ["--import-mode","importlib"],
+    "python.testing.autoTestDiscoverOnSaveEnabled": true,
+    "python.envFile": "${workspaceFolder}/.env",
+    "python-envs.defaultEnvManager": "ms-python.python:conda",
+    "python-envs.defaultPackageManager": "ms-python.python:conda",
+    "python-envs.pythonProjects": [],
 }

admin-backend/Dockerfile

@@ -19,7 +19,7 @@ COPY admin-backend/pyproject.toml admin-backend/poetry.lock ./
 RUN mkdir log && chmod 700 log
 RUN touch /app/admin-backend/log/logfile.log && chmod 600 /app/admin-backend/log/logfile.log
 
-RUN poetry config virtualenvs.create false &&\
+RUN poetry config virtualenvs.create false && \
     if [ "$dev" = "1" ]; then \
         poetry install --no-interaction --no-ansi --no-root --with dev; \
     else \

admin-backend/poetry.lock

@@ -0,0 +1,15 @@
+import sys
+import os
+from pathlib import Path
+
+# Add project root and specific directories to Python path
+project_root = Path(__file__).parent
+sys.path.insert(0, str(project_root))
+sys.path.insert(0, str(project_root / "admin-backend"))
+sys.path.insert(0, str(project_root / "rag-backend"))
+sys.path.insert(0, str(project_root / "document-extractor"))
+
+# point at each rag-core library's src folder so their packages (admin_api_lib, rag_core_api, etc.) are importable
+lib_root = project_root / "rag-core-library"
+for lib in ["admin-api-lib", "rag-core-api", "rag-core-lib", "extractor-api-lib"]:
+    sys.path.insert(0, str(lib_root / lib / "src"))

admin-backend/tests/dummy_test.py renamed to admin-backend/tests/dummy1_test.py

@@ -3,7 +3,7 @@ import { ref } from 'vue';
 import { DocumentModel } from "../../models/document.model.ts";
 import { ErrorType } from "../../models/error-type";
 import { UploadedDocument, mapToUploadDocument } from "../../models/uploaded-document.model";
-import { DocumentAPI } from "../document.api";
+import { DocumentAPI, ConfluenceConfig } from "../document.api";
 
 export const useDocumentsStore = defineStore('chat', () => {
     const uploadedDocuments = ref<UploadedDocument[]>([]);
@@ -52,11 +52,12 @@ export const useDocumentsStore = defineStore('chat', () => {
         }
     };
 
-    const loadConfluence = async () => {
+    const loadConfluence = async (config: ConfluenceConfig) => {
       isLoadingConfluence.value = true;
       error.value = null;
       try {
-        await DocumentAPI.loadConfluence();
+        // provide confluence configuration from frontend
+        await DocumentAPI.loadConfluence(config);
         await loadDocuments(); // Refresh the document list after uploading
       } catch(err) {
         if (err.response && err.response.status === 501) {

conftest.py

@@ -7,6 +7,15 @@ axios.defaults.auth = {
     password: import.meta.env.VITE_AUTH_PASSWORD
 };
 
+// confluence configuration interface
+export interface ConfluenceConfig {
+  spaceKey: string;
+  token: string;
+  url: string;
+  maxPages: number;
+  name: string;
+}
+
 export class DocumentAPI {
     static async loadDocuments(): Promise<DocumentModel[]> {
         try {
@@ -20,9 +29,9 @@ export class DocumentAPI {
     static async uploadDocument(file: File, onUploadProgress: (progressEvent: AxiosProgressEvent) => void): Promise<null> {
         try {
             const formData = new FormData();
-            formData.append('body', file);
+            formData.append('file', file);
 
-            const response = await axios.post<null>('/upload_documents', formData, {
+            const response = await axios.post<null>('/upload_file', formData, {
                 headers: {
                     'Content-Type': 'multipart/form-data'
                 },
@@ -35,9 +44,19 @@ export class DocumentAPI {
         }
     }
 
-    static async loadConfluence(): Promise<void> {
+    static async loadConfluence(config: ConfluenceConfig): Promise<void> {
         try {
-            await axios.post<void>('/load_confluence');
+            // convert config to list of key/value items for backend
+            const payload = [
+                { key: 'url', value: config.url },
+                { key: 'token', value: config.token },
+                { key: 'space_key', value: config.spaceKey },
+                { key: 'max_pages', value: String(config.maxPages) }
+            ];
+            // include required query parameters
+            await axios.post<void>('/upload_source', payload, {
+                params: { source_type: 'confluence', name: config.name }
+            });
         } catch(error) {
             this.handleError(error);
         }