diff --git a/.github/workflows/automation.yml b/.github/workflows/automation.yml index f45a692..e40acd0 100644 --- a/.github/workflows/automation.yml +++ b/.github/workflows/automation.yml @@ -64,12 +64,7 @@ jobs: run: npm ci - name: Run WDIO Tests - env: - WP_URL: ${{ secrets.WP_URL }} - WP_USERNAME: ${{ secrets.WP_USERNAME }} - WP_PASSWORD: ${{ secrets.WP_PASSWORD }} - OBOT_URL: ${{ secrets.OBOT_URL }} - GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }} + env: ${{ secrets }} run: | npm run wdio:byScenario npm run eval diff --git a/auto_eval.ts b/auto_eval.ts index e813e01..b71222d 100644 --- a/auto_eval.ts +++ b/auto_eval.ts @@ -44,7 +44,6 @@ interface GradeInfo { interface ToolData { responses?: string[]; - errors?: string[]; task_done?: boolean | null; failure_reason?: string[]; status?: string; @@ -118,10 +117,7 @@ async function enhanceReportWithEval( // Merge reasons const reasons: string[] = []; if (gradeInfo.reason) reasons.push(gradeInfo.reason); - if (toolData.errors?.length) reasons.push(...toolData.errors); - toolData.failure_reason = reasons; - delete toolData.errors; // Set status based on grading if (gradeInfo.result === "FAILURE") toolData.status = "Failure"; diff --git a/src/core/mcpFunc.ts b/src/core/mcpFunc.ts index d82a060..07b2878 100644 --- a/src/core/mcpFunc.ts +++ b/src/core/mcpFunc.ts @@ -68,7 +68,7 @@ export async function sendPromptValidateAndCollect(promptText: string, toolList: // Send and wait for reply const reply = await sendPromptAndWaitForReply(promptText); - await browser.pause(2000); + await browser.pause(10000); // Wait until a new message-content div appears await browser.waitUntil(async () => { @@ -98,28 +98,12 @@ export async function sendPromptValidateAndCollect(promptText: string, toolList: const currReply = promptReplies[await promptReplies.length - 1]; if (!currReply) throw new Error(`No reply container found even after waiting for prompt: "${promptText}"`); - // Validation regex - const successRegex = /(success|completed|connected|created|retrieved|posted|updated|closed|deleted|functioning|valid|available|ready to use)/i; - const failureRegex = /(not valid|failed|error|cannot access|do not have|insufficient|not available|required|troubleshooting)/i; - - const hasSuccess = successRegex.test(reply); - const hasFailure = failureRegex.test(reply); - - let errorMessage = ''; - if (!hasSuccess && !hasFailure) { - errorMessage = `No success or actionable failure detected in prompt #${index + 1} response.`; - } - - console.log(`Prompt #${index + 1}: Tools used: ${toolsTexts.length ? toolsTexts.join(', ') : 'None'} | Status: ${hasSuccess ? 'Success' : (hasFailure ? 'Failure' : 'Unknown')}`); - // Return data for reporting return { prompt: promptText, reply, replyElement: currReply, tools: toolsTexts, - status: hasSuccess ? 'Success' : (hasFailure ? 'Failure' : 'Unknown'), - error: errorMessage || null, }; } @@ -131,13 +115,13 @@ function maxStatus(s1: string, s2: string): string { export function aggregateToolResponses(promptResults: any[]) { const report: Record + tools: Record }> = {}; for (let i = 0; i < promptResults.length; i++) { const result = promptResults[i]; - const { prompt, tools, reply, status, error } = result; - if (!reply && !error) continue; + const { prompt, tools, reply } = result; + if (!reply) continue; const promptKey = `Prompt #${i + 1}`; @@ -153,14 +137,10 @@ export function aggregateToolResponses(promptResults: any[]) { for (const tool of toolsToUse) { if (!report[promptKey].tools[tool]) { - report[promptKey].tools[tool] = { responses: [], status: 'Unknown', errors: [] }; + report[promptKey].tools[tool] = { responses: []}; } if (reply) report[promptKey].tools[tool].responses.push(reply); - if (error) report[promptKey].tools[tool].errors.push(error); - - report[promptKey].tools[tool].status = - maxStatus(status, report[promptKey].tools[tool].status); } } @@ -170,7 +150,7 @@ export function aggregateToolResponses(promptResults: any[]) { export function saveMCPReport(mcpName: string, reportJson: any) { const folderName = `MCP Server Reports`; const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); - const fileName = `${mcpName}_MCP_Report_${timestamp}.json`; + const fileName = `${mcpName.toLowerCase().replace(/\s+/g, '_')}_MCP_Report_${timestamp}.json`; const dirPath = path.join(process.cwd(), folderName); const filePath = path.join(dirPath, fileName); diff --git a/src/core/selectors.ts b/src/core/selectors.ts index f51a671..d0e804d 100644 --- a/src/core/selectors.ts +++ b/src/core/selectors.ts @@ -102,6 +102,11 @@ const Selectors = { tableInput:'//pre[contains(@class, "whitespace-pre-wrap")]//span[contains(@class, "text-gray-500")]', showDetails2: '(//button[text()="Show Details"])[2]', showDetails3: '(//button[text()="Show Details"])[3]', + connectionsList: `//div[@class="flex flex-col"]`, + currentProjectButton: `//span[text()="Project"]/ancestor::button`, + createNewProjectButton: `//button[text()=" Create New Project"]`, + inputNewProjectName: '//input[@id="project-name"]', + saveButton: '//button[text()="Save"]', admin:{ oktaLogin:'//button[normalize-space(.//div) = "Sign in with Okta"]', @@ -210,22 +215,66 @@ const Selectors = { clickChatObot: '//button[normalize-space(text())="Chat"]', connectorbtn: '//p[normalize-space(text())="Connectors"]/following-sibling::button', mcpSearchInput: '//input[normalize-space(@placeholder)="Search by name..."]', - // selectMCPServer: '//p[normalize-space(text())="WordPress1"]', selectMCPServer: (option: string) => `//p[normalize-space(text())="${option}"]/ancestor::div[contains(@class, 'flex')]/button`, - wpSiteURL: '//input[normalize-space(@id)="WORDPRESS_SITE"]', - wpUsername: '//input[normalize-space(@id)="WORDPRESS_USERNAME"]', - wpPassword: '//input[normalize-space(@id)="WordPress App Password"]', btnClick: (option: string) => `//button[normalize-space(text())="${option}"]`, promptInput: '//div[@class="plaintext-editor text-md relative w-full flex-1 grow resize-none p-2 leading-8 outline-none"]', - // submitPrompt: '//button[@type="submit"]', - // obotInput: '//div[@class="ProseMirror editor"]', - gitlabToken: '//input[@name="GitLab Personal Access Token"]', - // messageContainer: "//div[contains(@class, 'flex-1') and contains(@class, 'flex-col') and contains(@class, 'justify-start') and contains(@class, 'gap-8')]", obotInput: "//div[contains(@class,'ProseMirror') and @contenteditable='true']", submitPrompt: '//button[@type="submit"]', lastBotReply: '//div[@class="message-content"]', - messageContainer: "//div[contains(@class, 'flex-1') and contains(@class, 'flex-col') and contains(@class, 'justify-start') and contains(@class, 'gap-8')]" - + messageContainer: "//div[contains(@class, 'flex-1') and contains(@class, 'flex-col') and contains(@class, 'justify-start') and contains(@class, 'gap-8')]", + wordpressMCP:{ + wpSiteURL: '//input[normalize-space(@id)="WORDPRESS_SITE"]', + wpUsername: '//input[normalize-space(@id)="WORDPRESS_USERNAME"]', + wpPassword: '//input[normalize-space(@id)="WordPress App Password"]', + }, + gitlabMCP:{ + gitlabToken: '//input[@name="GitLab Personal Access Token"]', + }, + bigQuery:{ + googleCloudProjectID: '//input[@id="GOOGLE_CLOUD_PROJECT"]', + googleCloudCredentials: '//input[@name="Google Application Credentials"]//following-sibling::div[1]', + }, + datadog:{ + datadogAPIKey: `//input[@id="Datadog API Key"]`, + datadogAPPKey: `//input[@id="Datadog App Key"]`, + }, + databricks:{ + utility:{ + workspaceHostname: `//input[@id="DATABRICKS_WORKSPACE_URL"]`, + functionCatalog: `//input[@id="DATABRICKS_FUNCTIONS_CATALOG"]`, + functionalSchema: `//input[@id="DATABRICKS_FUNCTIONS_SCHEMA"]`, + PAT: `//input[@id="Personal Access Token"]`, + }, + vector:{ + vectorCatalog: `//input[@id="DATABRICKS_VECTOR_SEARCH_CATALOG"]`, + vectorSchema: `//input[@id="DATABRICKS_VECTOR_SEARCH_SCHEMA"]`, + }, + genie: { + genieSpaceID: `//input[@id="DATABRICKS_GENIE_SPACE_ID"]` + } + }, + brave: { + braveAPIKey: `//input[@id="Brave API Key"]` + }, + chromaCloud: { + tenentID: `//input[@id="CHROMA_TENANT"]`, + DBName: `input[@id="CHROMA_DATABASE"]`, + APIKey: `//input[@id="Chroma Cloud API Key"]` + }, + fireCrawl: { + API_key: `//input[@id="Firecrawl API Key"]` + }, + gitMCP: { + urlLink: `//input[@id="url-manifest-url"]` + }, + redis: { + urlLink: `//input[@id="REDIS_URI"]` + }, + postman: { + hostURL: `//input[@id="HOST_URL"]`, + toolCOnfig: `//input[@id="TOOL_CONFIGURATION"]`, + postmanAPIKey: `//input[@id="Postman API Key"]`, + } } } export default Selectors; \ No newline at end of file diff --git a/src/data/bigquery_toolbox.MCP.json b/src/data/bigquery_toolbox.MCP.json new file mode 100644 index 0000000..701a509 --- /dev/null +++ b/src/data/bigquery_toolbox.MCP.json @@ -0,0 +1,20 @@ +{ + "prompts": [ + "Is BigQuery Toolbox connected?", + "Retrieve all columns from the table company_data.Employees where Salary is greater than 60000.", + "List the first 10 employees' first and last names from company_data.Employees.", + "Count the total number of employees in company_data.Employees.", + "Find the average salary of all employees in company_data.Employees.", + "Find all employees from company_data.Employees who were hired after January 1, 2023.", + "Get the total salary amount paid to all employees in company_data.Employees.", + "List employees from company_data.Employees with their email addresses ordered by HireDate descending.", + "Use service account credentials to query the company_data.Employees table.", + "Run SELECT * FROM company_data.Employees LIMIT 5.", + "Find the employee with the highest salary in company_data.Employees.", + "Find the employee with the lowest salary in company_data.Employees.", + "Find the number of employees hired per year from company_data.Employees." + ], + "tools": [ + "query" + ] +} diff --git a/src/data/brave_search.MCP.json b/src/data/brave_search.MCP.json new file mode 100644 index 0000000..c727c55 --- /dev/null +++ b/src/data/brave_search.MCP.json @@ -0,0 +1,25 @@ +{ + "prompts": [ + "Is Brave Search MCP server connected?", + "Search the web for 'best productivity apps 2025' and return the top 5 results.", + "Find web pages about 'remote work trends' published in the last 7 days.", + "Search for the latest news about 'AI regulation' and return the 5 most recent articles.", + "Find breaking news on 'electric vehicles' from the last 24 hours.", + "Search for images of 'modern home office setups' and return the top 5 results.", + "Find safe-for-work images of 'healthy breakfast ideas'.", + "Search for videos about 'machine learning tutorials' and return the top 3 results.", + "Find recent videos on '2025 tech conferences' published in the last month.", + "Find coffee shops near 'Denver, CO' with ratings and reviews.", + "Search for top-rated vegan restaurants in 'San Francisco'.", + "Summarize the top web results for 'benefits of meditation' with inline references.", + "Generate a concise summary of news articles about 'renewable energy investments'." + ], + "tools": [ + "brave_web_search", + "brave_news_search", + "brave_image_search", + "brave_video_search", + "brave_local_search", + "brave_summarizer" + ] +} \ No newline at end of file diff --git a/src/data/chroma_cloud.MCP.json b/src/data/chroma_cloud.MCP.json new file mode 100644 index 0000000..db1c58a --- /dev/null +++ b/src/data/chroma_cloud.MCP.json @@ -0,0 +1,32 @@ +{ + "prompts": [ + "Create a new collection called 'test_collection' using the default embedding function.", + "Add the following documents to a collection named 'test_collection': 'This is the first test document.' (ID: doc1), 'This is the second test document.' (ID: doc2).", + "Retrieve all documents from the collection 'test_collection'.", + "Get information about the collection 'test_collection'.", + "Get the number of documents in the collection 'test_collection'.", + "Peek at the first 3 documents in the collection 'test_collection'.", + "Query the collection 'test_collection' for documents similar to 'test document' and return the top 2 results.", + "Update the document with ID 'doc2' in 'test_collection' to have the content: 'This is the updated second test document.'", + "Delete the document with ID 'doc1' from the collection 'test_collection'.", + "Fork the collection 'test_collection' into a new collection called 'test_collection_fork'.", + "Rename the collection 'test_collection' to 'test_collection_renamed'.", + "List all collections in Chroma Cloud.", + "Delete the collection named 'test_collection_renamed'." + ], + "tools": [ + "chroma_create_collection", + "chroma_add_documents", + "chroma_get_documents", + "chroma_get_collection_info", + "chroma_get_collection_count", + "chroma_peek_collection", + "chroma_query_documents", + "chroma_update_documents", + "chroma_delete_documents", + "chroma_fork_collection", + "chroma_modify_collection", + "chroma_list_collections", + "chroma_delete_collection" + ] +} \ No newline at end of file diff --git a/src/data/databrick_genie.MCP.json b/src/data/databrick_genie.MCP.json new file mode 100644 index 0000000..1accccc --- /dev/null +++ b/src/data/databrick_genie.MCP.json @@ -0,0 +1,20 @@ +{ + "prompts": [ + "Connect to the Databricks Genie Spaces MCP server.", + "List all available tools under Databricks Genie Spaces.", + "Attempt to fetch data without specifying any parameters.", + "What is the total sales for Q1 2024?", + "Give me insights on the trend of product sales over the last year.", + "How many new customers did we acquire in the last 3 months?", + "Which products had the highest sales last week in region A?", + "Run a serverless SQL query to fetch the top 10 customers by revenue.", + "Use serverless SQL to aggregate sales data by product category.", + "Run a complex serverless SQL query on a large dataset.", + "Show me the trend of website traffic for the last 6 months.", + "Give me the top 5 sales-performing regions over the past quarter.", + "Request data insights across multiple datasets concurrently.", + "Run multiple BI queries about different products and regions at the same time.", + "Search for product sales data across multiple years and regions." + ], + "tools": [] +} diff --git a/src/data/databrick_unity.MCP.json b/src/data/databrick_unity.MCP.json new file mode 100644 index 0000000..c0f42c3 --- /dev/null +++ b/src/data/databrick_unity.MCP.json @@ -0,0 +1,17 @@ +{ + "prompts": [ + "Is Databricks Unity Catalog Functions connected?", + "Execute the function get_user_activity from analytics_catalog.user_schema catalog with the following parameters: start_date='2025-10-01', end_date='2025-10-10'.", + "Execute the function current_timestamp from system.default catalog with no parameters.", + "Execute the function date_add from system.default catalog with the following parameters: start_date='2025-10-01', days=5.", + "Run the function get_total_sales from sales_catalog.monthly_reports with the parameter month='September'.", + "Execute the function get_user_activity from analytics_catalog.user_schema with the parameters start_date='invalid_date' and end_date='2025-10-10' to test error handling.", + "Run the function calculate_sales with an invalid parameter region_code='XYZ' that does not exist in the valid list.", + "Execute the function concat from system.default catalog with the following parameters: str1='Hello', str2='World'.", + "Execute the function non_existent_function from default.test_schema catalog to test function not found error.", + "Execute the function current_date from system.default catalog with invalid parameter test_param='xyz' to test parameter validation.", + "SELECT analytics_catalog.user_schema.get_user_activity('2025-10-01', '2025-10-10');", + "Run the SQL query: SELECT analytics_catalog.user_schema.get_user_activity('2025-10-01', '2025-10-10');" + ], + "tools": [] +} \ No newline at end of file diff --git a/src/data/databrick_vector.MCP.json b/src/data/databrick_vector.MCP.json new file mode 100644 index 0000000..7aeb54d --- /dev/null +++ b/src/data/databrick_vector.MCP.json @@ -0,0 +1,25 @@ +{ + "prompts": [ + "Connect to the Databricks Vector Search MCP server.", + "List all available tools under Databricks Vector Search.", + "Fetch data from Databricks using the vector model without specifying parameters.", + "Search for documents related to 'machine learning models'.", + "Perform semantic search using the vector embedding for the phrase 'neural networks in deep learning'.", + "Find similar items to the embedding for the phrase 'cloud computing infrastructure'.", + "Search for articles about 'distributed systems' and return results that are more relevant to 'data replication in databases'.", + "Find similar items to a non-existent or irrelevant embedding.", + "List all available vector search indexes in the system.", + "Create a new vector search index for the dataset 'customer reviews'.", + "Modify the index 'product_reviews' by adding a new field for 'review_rating'.", + "Delete the 'outdated_documents' index from the search database.", + "Create an index without specifying a dataset.", + "Use serverless compute to run a semantic search over the 'tech_education' dataset.", + "Check if the serverless compute is being utilized for this search operation.", + "Perform a search on a very large dataset that may result in a timeout.", + "Run the same semantic search query twice: 'data analytics tools for 2024'.", + "Search for content related to 'AWS EC2' from documents published after 2020.", + "Fetch results for 'cloud services' and include 'cloud_security' tag.", + "Perform a semantic search on the entire AWS documentation with a high query volume." + ], + "tools": [] +} diff --git a/src/data/datadog.MCP.json b/src/data/datadog.MCP.json new file mode 100644 index 0000000..c0076d7 --- /dev/null +++ b/src/data/datadog.MCP.json @@ -0,0 +1,51 @@ +{ + "prompts": [ + "Is Datadog tool connected?", + "List all dashboards in my Datadog account.", + "Fetch details for the first dashboard returned by the dashboard list.", + "Get the count of active hosts in Datadog.", + "List all hosts currently registered in Datadog.", + "Extract all unique service names from Datadog logs.", + "Query logs from the last 24 hours in Datadog.", + "Search for logs with status:error in Datadog.", + "List all incidents currently present in Datadog.", + "Fetch details for the most recent incident in Datadog.", + "List all monitors configured in Datadog.", + "List all monitors currently in alert state in Datadog.", + "Schedule a downtime for a test monitor in Datadog.", + "List all currently scheduled downtimes in Datadog.", + "Cancel a downtime by its ID in Datadog.", + "Mute a test host in Datadog.", + "Unmute the test host in Datadog.", + "List all RUM applications in Datadog.", + "Query recent RUM events in Datadog.", + "Group RUM events by application.name and count them in Datadog.", + "Get page performance metrics for RUM data in Datadog.", + "Get the RUM page waterfall for a given application and session in Datadog.", + "Query the system.cpu.user metric for the last hour in Datadog.", + "List recent APM traces for a test service in Datadog." + ], + "tools": [ + "cancel_downtime", + "get_active_hosts_count", + "get_all_services", + "get_dashboard", + "get_incident", + "get_logs", + "get_monitors", + "get_rum_applications", + "get_rum_events", + "get_rum_grouped_event_count", + "get_rum_page_performance", + "get_rum_page_waterfall", + "list_dashboards", + "list_downtimes", + "list_hosts", + "list_incidents", + "list_traces", + "mute_host", + "query_metrics", + "schedule_downtime", + "unmute_host" +] +} \ No newline at end of file diff --git a/src/data/deepwiki.MCP.json b/src/data/deepwiki.MCP.json new file mode 100644 index 0000000..4e892f9 --- /dev/null +++ b/src/data/deepwiki.MCP.json @@ -0,0 +1,13 @@ +{ + "prompts": [ + "Is DeepWiki MCP tool connected?", + "Show me the documentation structure for the GitHub repository facebook/react using read_wiki_structure.", + "Display the main documentation contents for the GitHub repository facebook/react using read_wiki_contents.", + "What is the purpose of the useState hook in the facebook/react repository using ask_question?" + ], + "tools": [ + "read_wiki_structure", + "read_wiki_contents", + "ask_question" + ] +} \ No newline at end of file diff --git a/src/data/exa_search.MCP.Json b/src/data/exa_search.MCP.Json new file mode 100644 index 0000000..51568b3 --- /dev/null +++ b/src/data/exa_search.MCP.Json @@ -0,0 +1,11 @@ +{ + "prompts": [ + "Is Exa Search tool connected?", + "Show me Python code examples for reading and writing JSON files using the built-in json module with get_code_context_exa.", + "What are the most important new features in React 19 as of November 2025 using web_search_exa?" + ], + "tools": [ + "get_code_context_exa", + "web_search_exa" + ] +} diff --git a/src/data/firecrawl.MCP.json b/src/data/firecrawl.MCP.json new file mode 100644 index 0000000..3faebfc --- /dev/null +++ b/src/data/firecrawl.MCP.json @@ -0,0 +1,20 @@ +{ + "prompts": [ + "Is Firecrawl tool connected?", + "Validate the connection to the Firecrawl service.", + "Search the web for the official homepage of 'Obot AI'.", + "Scrape the main heading (h1) from https://obot.ai/ in markdown format.", + "List the first 5 URLs found on https://obot.ai/.", + "Crawl https://obot.ai/ with a depth of 0 and a limit of 1 page.", + "Extract the page title and the first sentence from https://obot.ai/.", + "Check the status of the most recent crawl job." + ], + "tools": [ + "firecrawl_search", + "firecrawl_scrape", + "firecrawl_map", + "firecrawl_crawl", + "firecrawl_extract", + "firecrawl_check_crawl_status" + ] +} \ No newline at end of file diff --git a/src/data/gitmcp.MCP.json b/src/data/gitmcp.MCP.json new file mode 100644 index 0000000..bf20dce --- /dev/null +++ b/src/data/gitmcp.MCP.json @@ -0,0 +1,16 @@ +{ + "prompts": [ + "What is the owner and repo for the library 'react'?", + "Fetch the documentation for the repository 'facebook/react'.", + "Search the documentation in 'facebook/react' for the term 'hooks'.", + "Search the code in 'facebook/react' for the keyword 'useState'.", + "Fetch the content from this URL: https://raw.githubusercontent.com/facebook/react/main/README.md" + ], + "tools": [ + "gitmcpMatchCommonLibsOwnerRepoMapping", + "gitmcpFetchGenericDocumentation", + "gitmcpSearchGenericDocumentation", + "gitmcpSearchGenericCode", + "gitmcpFetchGenericUrlContent" + ] +} \ No newline at end of file diff --git a/src/data/postman.MCP.json b/src/data/postman.MCP.json new file mode 100644 index 0000000..f1e3480 --- /dev/null +++ b/src/data/postman.MCP.json @@ -0,0 +1,139 @@ +{ + "prompts": [ + "Create a new collection named 'Dummy Collection for Tool Testing' in workspace 1213ab29-47cc-4f5d-810c-0a307b54a9ce.", + "Add a comment 'This is a test comment' to the collection with ID fe28eb66-9a9d-4d48-821e-cbba279181e5.", + "Create a folder named 'Test Folder' in the collection with ID fe28eb66-9a9d-4d48-821e-cbba279181e5.", + "Fork the collection with ID fe28eb66-9a9d-4d48-821e-cbba279181e5 into workspace 1213ab29-47cc-4f5d-810c-0a307b54a9ce and label it 'Test Fork'.", + "Add a GET request named 'Test Request' to the collection with ID fe28eb66-9a9d-4d48-821e-cbba279181e5.", + "Add a response to the request with ID 85bb120b-5802-b5e8-a16c-ef48d25c1c42 in collection fe28eb66-9a9d-4d48-821e-cbba279181e5. (Note: Response ID creation may require manual setup)", + "Create a new environment named 'Test Environment' in workspace 1213ab29-47cc-4f5d-810c-0a307b54a9ce.", + "Add a comment 'Test folder comment' to folder a818298d-d548-21bd-eef6-d18f85f9811f in collection fe28eb66-9a9d-4d48-821e-cbba279181e5.", + "Create a mock server for collection 31616058-fe28eb66-9a9d-4d48-821e-cbba279181e5 in workspace 1213ab29-47cc-4f5d-810c-0a307b54a9ce.", + "Add a comment 'Test request comment' to request 85bb120b-5802-b5e8-a16c-ef48d-25c1c42 in collection fe28eb66-9a9d-4d48-821e-cbba279181e5.", + "Create a new OpenAPI 3.0 spec named 'Test Spec' in workspace 1213ab29-47cc-4f5d-810c-0a307b54a9ce.", + "Add a file named 'test.yaml' to spec 8d7ad57a-8d7b-4ad3-b1e0-df901d3903e7 with the content 'openapi: 3.0.0'.", + "Create a new workspace named 'Test Workspace' of type 'personal'.", + "Update the name of collection fe28eb66-9a9d-4d48-821e-cbba279181e5 to 'Updated Test Collection'.", + "Update the variable 'baseUrl' in environment a4fe7252-9078-4d08-8cb3-8a812468c78f to 'https://example.com'.", + "Run the monitor with ID 1f0be5c4-9f53-4240-8b8f-75bade8b5686 now.", + "Publish documentation for collection fe28eb66-9a9d-4d48-821e-cbba279181e5 with the default layout.", + "Unpublish documentation for collection fe28eb66-9a9d-4d48-821e-cbba279181e5.", + "Publish the mock server with ID c35efb81-cbcf-44b2-8602-eacf84ae877d.", + "Unpublish the mock server with ID c35efb81-cbcf-44b2-8602-eacf84ae877d.", + "Update the monitor 1f0be5c4-9f53-4240-8b8f-75bade8b5686 to run every 30 minutes.", + "Update the name of spec 8d7ad57a-8d7b-4ad3-b1e0-df901d3903e7 to 'Updated Spec Name'.", + "Update the file 'test.yaml' in spec 8d7ad57a-8d7b-4ad3-b1e0-df901d3903e7 with new content.", + "Update the name of workspace 1213ab29-47cc-4f5d-810c-0a307b54a9ce to 'Updated Workspace'.", + "Update tags for workspace 1213ab29-47cc-4f5d-810c-0a307b54a9ce to include 'test-tag'.", + "Update the global variable 'apiKey' in workspace 1213ab29-47cc-4f5d-810c-0a307b54a9ce to '12345'." + ], + "tools": [ + "createCollection", + "createCollectionComment", + "createCollectionFolder", + "createCollectionFork", + "createCollectionRequest", + "createCollectionResponse", + "createEnvironment", + "createFolderComment", + "createMock", + "createMonitor", + "createRequestComment", + "createResponseComment", + "createSpec", + "createSpecFile", + "createWorkspace", + "deleteApiCollectionComment", + "deleteCollection", + "deleteCollectionComment", + "deleteCollectionFolder", + "deleteCollectionRequest", + "deleteCollectionResponse", + "deleteEnvironment", + "deleteFolderComment", + "deleteMock", + "deleteMonitor", + "deletePanElementOrFolder", + "deleteRequestComment", + "deleteResponseComment", + "deleteSpec", + "deleteSpecFile", + "deleteWorkspace", + "duplicateCollection", + "generateCollection", + "generateSpecFromCollection", + "getAllElementsAndFolders", + "getAllPanAddElementRequests", + "getAllSpecs", + "getAsyncSpecTaskStatus", + "getAuthenticatedUser", + "getCollection", + "getCollectionComments", + "getCollectionFolder", + "getCollectionForks", + "getCollectionRequest", + "getCollectionResponse", + "getCollectionTags", + "getCollectionUpdatesTasks", + "getCollections", + "getCollectionsForkedByUser", + "getDuplicateCollectionTaskStatus", + "getEnvironment", + "getEnvironments", + "getFolderComments", + "getGeneratedCollectionSpecs", + "getMock", + "getMocks", + "getMonitor", + "getMonitors", + "getRequestComments", + "getResponseComments", + "getSourceCollectionStatus", + "getSpec", + "getSpecCollections", + "getSpecDefinition", + "getSpecFile", + "getSpecFiles", + "getStatusOfAnAsyncApiTask", + "getTaggedEntities", + "getWorkspace", + "getWorkspaceGlobalVariables", + "getWorkspaceTags", + "getWorkspaces", + "mergeCollectionFork", + "patchCollection", + "patchEnvironment", + "postPanElementOrFolder", + "publishDocumentation", + "publishMock", + "pullCollectionChanges", + "putCollection", + "putEnvironment", + "resolveCommentThread", + "runMonitor", + "syncCollectionWithSpec", + "syncSpecWithCollection", + "transferCollectionFolders", + "transferCollectionRequests", + "transferCollectionResponses", + "unpublishDocumentation", + "unpublishMock", + "updateApiCollectionComment", + "updateCollectionComment", + "updateCollectionFolder", + "updateCollectionRequest", + "updateCollectionResponse", + "updateCollectionTags", + "updateFolderComment", + "updateMock", + "updateMonitor", + "updatePanElementOrFolder", + "updateRequestComment", + "updateResponseComment", + "updateSpecFile", + "updateSpecProperties", + "updateWorkspace", + "updateWorkspaceGlobalVariables", + "updateWorkspaceTags" + ] +} \ No newline at end of file diff --git a/src/data/redis.MCP.json b/src/data/redis.MCP.json new file mode 100644 index 0000000..1c120d9 --- /dev/null +++ b/src/data/redis.MCP.json @@ -0,0 +1,21 @@ +{ + "prompts": [ + "Show me the list of connected Redis clients.", + "Create a vector similarity index named 'test_vector_index' on hashes with prefix 'testdoc:', vector field 'vector', dimension 4, and distance metric 'COSINE'. Then show the schema and info for the index, and list all indexes in the database.", + "How many keys are currently stored in the Redis database? Scan and return all keys matching the pattern 'test*' and scan for keys matching the pattern 'test*' (first batch).", + "Set the key 'testkey' to the value 'testvalue', get its value, set it to expire in 60 seconds, check its type, and then delete it.", + "Create hash 'testhash' with field 'field1' set to 'value1', check if 'field1' exists, get its value, get all fields and values, delete 'field1', and set it again.", + "Store the vector [0.1, 0.2, 0.3, 0.4] in the field 'vector' of hash 'testdoc:1', get the vector from hash 'testdoc:1', and search for the 2 nearest neighbors to the vector [0.1, 0.2, 0.3, 0.4] in index 'test_vector_index'.", + "Set the root JSON value of key 'testjson' to {\"foo\": \"bar\"}, get the root JSON value, and then delete it.", + "Push 'item1' onto the left and 'item2' onto the right of the list 'testlist', get all elements, get the length, remove and return the first and last elements.", + "Add 'member1' and 'member2' to the set 'testset', get all members, and remove 'member1'.", + "Add the member 'zmember1' with score 1.5 and 'zmember2' with score 2.5 to the sorted set 'testzset', get all members (with scores), and remove 'zmember1'.", + "Add two entries with field 'foo' and values 'bar' and 'baz' to the stream 'teststream', read the first 2 entries, and delete the entry with ID '123-0'.", + "Publish the message 'hello world' to the channel 'testchannel', subscribe and unsubscribe from the channel.", + "Set the key 'oldkey' to the value 'somevalue' and rename it to 'newkey'.", + "Show me the default Redis server info." + ], + "tools": [ + "client_list", "create_vector_index_hash", "get_index_info", "get_indexes", "dbsize", "scan_all_keys", "scan_keys", "set", "get", "expire", "type", "delete", "hset", "hexists", "hget", "hgetall", "hdel", "set_vector_in_hash", "get_vector_from_hash", "vector_search_hash", "json_set", "json_get", "json_del", "lpush", "rpush", "lrange", "llen", "lpop", "rpop", "sadd", "smembers", "srem", "zadd", "zrange", "zrem", "xadd", "xrange", "xdel", "publish", "subscribe", "unsubscribe", "rename", "info" + ] +} \ No newline at end of file diff --git a/src/data/tavily_search.MCP.json b/src/data/tavily_search.MCP.json new file mode 100644 index 0000000..e40dc1a --- /dev/null +++ b/src/data/tavily_search.MCP.json @@ -0,0 +1,15 @@ +{ + "prompts": [ + "Is Tavily Search tool connected?", + "Search for the latest advancements in AI using tavily_search.", + "Extract the main content from https://obot.ai/ using tavily_extract and summarize the result.", + "Crawl the first 3 pages of https://docs.python.org/3/ using tavily_crawl and summarize the topics.", + "Map the structure of https://developer.mozilla.org/ using tavily_map to list main documentation sections." + ], + "tools": [ + "tavily_search", + "tavily_extract", + "tavily_crawl", + "tavily_map" + ] +} \ No newline at end of file diff --git a/src/features/mcpServer.feature b/src/features/mcpServer.feature index 20459b5..19c9816 100644 --- a/src/features/mcpServer.feature +++ b/src/features/mcpServer.feature @@ -1,22 +1,34 @@ -Feature: Connecte MCP server on Obot +Feature: Connect MCP servers on Obot Background: Navigate to Obot Given I setup context for assertion - When User navigates the Obot main login page - Then User open chat Obot + When User navigates to the Obot main login page + Then User opens chat Obot + And User creates a new Project with no existing connections - Scenario: Validate Wordpress sequential prompts on Obot - When User open MCP connector page - And User select "WordPress" MCP server - And User select "Connect To Server" button - And User connect to the WordPress1 MCP server - When User sends prompts to Obot AI chat for "Wordpress" MCP server - Then All prompts results should be validated and report generated for selected "Wordpress" MCP Server + Scenario Outline: Validate sequential prompts on Obot + When User opens the MCP connector page + And User selects "" MCP server + And User selects "Connect To Server" button + And User connects to the "" MCP server + When User sends prompts to Obot AI chat for "" MCP server + Then All prompt results should be validated and a report generated for the selected "" MCP server - Scenario: Validate GitLab sequential prompts on Obot - When User open MCP connector page - And User select "GitLab" MCP server - And User select "Connect To Server" button - And User connect to the GitLab MCP server - When User sends prompts to Obot AI chat for "Gitlab" MCP server - Then All prompts results should be validated and report generated for selected "Gitlab" MCP Server \ No newline at end of file + Examples: + | ServerName | ConnectionName | PromptName | ReportName | + | WordPress | WordPress | WordPress | WordPress | + | GitLab | GitLab | GitLab | GitLab | + | BigQuery Toolbox | BigQuery | BigQuery Toolbox | BigQuery Toolbox | + # | Datadog | Datadog | Datadog | Datadog | + | Databricks Unity Catalog Functions | Databricks Unity Catalog Functions | Databrick Unity | Databrick Unity | + | Databricks Genie Spaces | Databricks Genie Space | Databrick Genie | Databrick Genie | + | Databricks Vector Search | Databricks Vector Space | Databrick Vector | Databrick Vector | + | Brave Search | Brave Search | Brave Search | Brave Search | + | Chroma Cloud | Chroma Cloud | Chroma Cloud | Chroma Cloud | + | Firecrawl | Firecrawl | Firecrawl | Firecrawl | + | GitMCP | GitMCP | GitMCP | GitMCP | + | Redis | Redis | Redis | Redis | + # | Postman | Postman | Postman | Postman | + | Tavily Search | Tavily Search | Tavily Search | Tavily Search | + | Exa Search | Exa Search | Exa Search | Exa Search | + | DeepWiki | DeepWiki | DeepWiki | DeepWiki | \ No newline at end of file diff --git a/src/steps/mcpServer.step.ts b/src/steps/mcpServer.step.ts index 86e5bd0..c7841cd 100644 --- a/src/steps/mcpServer.step.ts +++ b/src/steps/mcpServer.step.ts @@ -1,26 +1,27 @@ import { When, Then, Given } from "@wdio/cucumber-framework"; import Selectors from "../core/selectors"; import { clickToElement,isElementDisplayed,slowInputFilling} from "../core/func"; -import { LONG_PAUSE, SHORT_PAUSE } from "../core/timeouts"; +import { LONG_PAUSE, MEDIUM_PAUSE, SHORT_PAUSE } from "../core/timeouts"; import { aggregateToolResponses, saveMCPReport, sendPromptValidateAndCollect } from "../core/mcpFunc"; import path from 'path'; import { promises as fs } from 'fs'; +import { generateProjectName } from '../utils/projectName.model'; -Given(/^User navigates the Obot main login page$/, async() => { +Given(/^User navigates to the Obot main login page$/, async() => { const url = process.env.OBOT_URL ; await browser.url(url); }); -Then(/^User open chat Obot$/, async () => { +Then(/^User opens chat Obot$/, async () => { await clickToElement(Selectors.MCP.navigationbtn); await clickToElement(Selectors.MCP.clickChatObot); }); -When(/^User open MCP connector page$/, async () => { +When(/^User opens the MCP connector page$/, async () => { await clickToElement(Selectors.MCP.connectorbtn); }); -Then(/^User select "([^"]*)" MCP server$/, async (MCPServer) => { +Then(/^User selects "([^"]*)" MCP server$/, async (MCPServer) => { await slowInputFilling(Selectors.MCP.mcpSearchInput, MCPServer); await isElementDisplayed(Selectors.MCP.selectMCPServer(MCPServer), LONG_PAUSE); // Wait until matching elements appear @@ -35,18 +36,10 @@ Then(/^User select "([^"]*)" MCP server$/, async (MCPServer) => { await browser.pause(SHORT_PAUSE); }); -Then(/^User select "([^"]*)" button$/, async (Button) => { +Then(/^User selects "([^"]*)" button$/, async (Button) => { await isElementDisplayed(Selectors.MCP.btnClick(Button),SHORT_PAUSE); await clickToElement(Selectors.MCP.btnClick(Button)); }); - -Then(/^User connect to the WordPress1 MCP server$/, async () => { - await slowInputFilling(Selectors.MCP.wpSiteURL,process.env.WP_URL); - await slowInputFilling(Selectors.MCP.wpUsername,process.env.WP_USERNAME); - await slowInputFilling(Selectors.MCP.wpPassword, process.env.WP_PASSWORD); - await clickToElement(Selectors.MCP.btnClick("Launch")); - await browser.pause(LONG_PAUSE*2); -}); Then(/^User asks obot "([^"]*)"$/, async (prompt) => { await slowInputFilling(Selectors.MCP.obotInput, prompt); @@ -54,14 +47,8 @@ Then(/^User asks obot "([^"]*)"$/, async (prompt) => { await browser.pause(LONG_PAUSE); }); -Then(/^User connect to the GitLab MCP server$/, async () => { - await slowInputFilling(Selectors.MCP.gitlabToken,process.env.GITLAB_TOKEN); - await clickToElement(Selectors.MCP.btnClick("Launch")); - await browser.pause(LONG_PAUSE); -}); - When(/^User sends prompts to Obot AI chat for "([^"]*)" MCP server$/, { timeout: 15 * 60 * 1000 }, async function(serverName: string) { - const jsonPath = path.resolve(process.cwd(), 'src', 'data', `${serverName.toLowerCase()}.MCP.json`); + const jsonPath = path.resolve(process.cwd(), 'src', 'data', `${serverName.toLowerCase().replace(/\s+/g, '_')}.MCP.json`); const data = await fs.readFile(jsonPath, 'utf-8'); const { prompts, tools } = JSON.parse(data); @@ -79,7 +66,7 @@ When(/^User sends prompts to Obot AI chat for "([^"]*)" MCP server$/, { timeout: } }); -Then(/^All prompts results should be validated and report generated for selected "([^"]*)" MCP Server$/, async function(serverName: string) { +Then(/^All prompt results should be validated and a report generated for the selected "(.*)" MCP server$/, async function(serverName: string) { const report = aggregateToolResponses(this.promptResults); saveMCPReport(serverName, report); @@ -89,3 +76,105 @@ Then(/^All prompts results should be validated and report generated for selected } }); +Then(/^User connects to the "(.*)" MCP server$/, async (mcpServer: string) => { + switch (mcpServer.toLowerCase()) { + case 'wordpress': + await slowInputFilling(Selectors.MCP.wordpressMCP.wpSiteURL, process.env.WP_URL); + await slowInputFilling(Selectors.MCP.wordpressMCP.wpUsername, process.env.WP_USERNAME); + await slowInputFilling(Selectors.MCP.wordpressMCP.wpPassword, process.env.WP_PASSWORD); + break; + + case 'gitlab': + await slowInputFilling(Selectors.MCP.gitlabMCP.gitlabToken, process.env.GITLAB_TOKEN); + break; + + case 'bigquery': + await slowInputFilling(Selectors.MCP.bigQuery.googleCloudProjectID, process.env.BQ_PROJECTID); + await $(Selectors.MCP.bigQuery.googleCloudCredentials).setValue(process.env.BQ_APP_CREDS); + break; + + case 'datadog': + await slowInputFilling(Selectors.MCP.datadog.datadogAPIKey, process.env.DD_API_KEY); + await slowInputFilling(Selectors.MCP.datadog.datadogAPPKey, process.env.DD_APP_KEY); + break; + + case 'databricks utility': + await slowInputFilling(Selectors.MCP.databricks.utility.workspaceHostname, process.env.DB_UTILITY_WORKSPACENAME); + await slowInputFilling(Selectors.MCP.databricks.utility.functionCatalog, process.env.DB_UTILITY_FUNCATALOG || 'workspace'); + await slowInputFilling(Selectors.MCP.databricks.utility.functionalSchema, process.env.DB_UTILITY_FUNSCHEMA || 'information_schema'); + await slowInputFilling(Selectors.MCP.databricks.utility.PAT, process.env.DB_UTILITY_PAT); + break; + + case 'databricks vector space': + await slowInputFilling(Selectors.MCP.databricks.utility.workspaceHostname, process.env.DB_UTILITY_WORKSPACENAME); + await slowInputFilling(Selectors.MCP.databricks.vector.vectorCatalog, process.env.DB_UTILITY_FUNCATALOG || 'workspace'); + await slowInputFilling(Selectors.MCP.databricks.vector.vectorSchema, process.env.DB_UTILITY_FUNSCHEMA || 'information_schema'); + await slowInputFilling(Selectors.MCP.databricks.utility.PAT, process.env.DB_UTILITY_PAT); + break; + + case 'databricks genie space': + await slowInputFilling(Selectors.MCP.databricks.utility.workspaceHostname, process.env.DB_UTILITY_WORKSPACENAME); + await slowInputFilling(Selectors.MCP.databricks.genie.genieSpaceID, process.env.DB_GENIE_ID); + await slowInputFilling(Selectors.MCP.databricks.utility.PAT, process.env.DB_UTILITY_PAT); + break; + + case 'brave search': + await slowInputFilling(Selectors.MCP.brave.braveAPIKey, process.env.BRAVE_API_KEY); + break; + + case 'chroma cloud': + await slowInputFilling(Selectors.MCP.chromaCloud.tenentID, process.env.CHROMA_TENENT_ID); + await slowInputFilling(Selectors.MCP.chromaCloud.DBName, process.env.CHROMA_DB_NAME || 'obot-test'); + await slowInputFilling(Selectors.MCP.chromaCloud.APIKey, process.env.CHROMA_API_KEY); + break; + + case 'firecrawl': + await slowInputFilling(Selectors.MCP.fireCrawl.API_key, process.env.FIRECRAWL_API_KEY); + break; + + case 'gitmcp': + await slowInputFilling(Selectors.MCP.gitMCP.urlLink, "https://gitmcp.io/docs"); + break; + + case 'redis': + await slowInputFilling(Selectors.MCP.redis.urlLink, process.env.REDIS_URL); + break; + + case 'tavily search': + await slowInputFilling('//input[@id="Tavily API Key"]', process.env.TAVILY_API_KEY); + break; + + case 'exa search': + await slowInputFilling('//input[@id="Exa API Key"]', process.env.TAVILY_API_KEY); + break; + + case 'postman': + await slowInputFilling(Selectors.MCP.postman.hostURL, 'https://mcp.postman.com'); + await slowInputFilling(Selectors.MCP.postman.toolCOnfig, 'mcp'); + await slowInputFilling(Selectors.MCP.postman.postmanAPIKey, process.env.POSTMAN_API_KEY); + break; + + case 'deepwiki': + break; + + default: + throw new Error(`Unknown MCP Server: ${mcpServer}`); + } + + await clickToElement(Selectors.MCP.btnClick('Launch')); + await browser.pause(LONG_PAUSE * 2); +}); + +When("User creates a new Project with no existing connections", async () => { + await browser.pause(MEDIUM_PAUSE); + const connectionElement = await $(Selectors.connectionsList).isDisplayed(); + const projectName = generateProjectName(); + if(connectionElement) { + await clickToElement(Selectors.currentProjectButton); + await clickToElement(Selectors.createNewProjectButton); + await slowInputFilling(Selectors.inputNewProjectName, projectName); + await clickToElement(Selectors.saveButton); + } + await browser.pause(MEDIUM_PAUSE); + await browser.refresh(); +}) \ No newline at end of file diff --git a/src/utils/projectName.model.ts b/src/utils/projectName.model.ts new file mode 100644 index 0000000..c9a9023 --- /dev/null +++ b/src/utils/projectName.model.ts @@ -0,0 +1,12 @@ +import { faker } from '@faker-js/faker'; + +/** + * Generate a random project name + * Format: Automation-Obot-MCP-XXXXXX + */ +export function generateProjectName(): string { + const randomNumber = faker.number.int({ min: 100000, max: 999999 }); + return `Automation-Obot-MCP-${randomNumber}`; +} + +console.log(generateProjectName()); \ No newline at end of file diff --git a/wdio.conf.ts b/wdio.conf.ts index a6cf302..7396897 100644 --- a/wdio.conf.ts +++ b/wdio.conf.ts @@ -182,7 +182,7 @@ export const config: WebdriverIO.Config = { // add cucumber tags to feature or scenario name tagsInTitle: false, // timeout for step definitions - timeout: 120000, + timeout: 180000, }, ...hooks, };