Improve chatting with project (#1644)

* Add project context icon

* Allow chatting without selecting elements

* Better tool use

* Read files

* Add Onlook tool

* Clean up

Author: Kitenite

apps/studio/src/lib/editor/engine/chat/context.ts

@@ -24,15 +24,23 @@ export class ChatContext {
             () => this.editorEngine.elements.selected,
             () => this.getChatContext().then((context) => (this.context = context)),
         );
+        reaction(
+            () => this.projectsManager.project?.folderPath,
+            (folderPath) => {
+                if (folderPath) {
+                    this.getChatContext().then((context) => (this.context = context));
+                }
+            },
+        );
     }
 
     async getChatContext(): Promise<ChatMessageContext[]> {
         const selected = this.editorEngine.elements.selected;
-        if (selected.length === 0) {
-            return [];
-        }
         const fileNames = new Set<string>();
-        const highlightedContext = await this.getHighlightedContext(selected, fileNames);
+        let highlightedContext: HighlightMessageContext[] = [];
+        if (selected.length) {
+            highlightedContext = await this.getHighlightedContext(selected, fileNames);
+        }
         const fileContext = await this.getFileContext(fileNames);
         const imageContext = await this.getImageContext();
         const projectContext = await this.getProjectContext();

apps/studio/src/lib/editor/engine/chat/index.ts

@@ -111,13 +111,13 @@ export class ChatManager {
             messages,
             requestType,
         );
-        this.stream.clear();
-        this.isWaiting = false;
         if (res) {
             this.handleChatResponse(res, requestType);
         } else {
             console.error('No stream response found');
         }
+        this.stream.clear();
+        this.isWaiting = false;
         sendAnalytics('receive chat response');
     }
 

apps/studio/src/routes/editor/EditPanel/ChatTab/ChatInput.tsx

@@ -295,10 +295,7 @@ export const ChatInput = observer(() => {
                     disabled={disabled}
                     placeholder={
                         disabled
-                            ? projectsManager.runner?.isRunning ||
-                              projectsManager.runner?.isStarting
-                                ? t('editor.panels.edit.tabs.chat.emptyState')
-                                : t('editor.panels.edit.tabs.chat.emptyStateStart')
+                            ? t('editor.panels.edit.tabs.chat.emptyState')
                             : t('editor.panels.edit.tabs.chat.input.placeholder')
                     }
                     className={cn(

apps/studio/src/routes/editor/EditPanel/ChatTab/ContextPills/helpers.tsx

@@ -1,8 +1,9 @@
 import { getTruncatedFileName } from '@/lib/utils';
-import { type ChatMessageContext } from '@onlook/models/chat';
+import { MessageContextType, type ChatMessageContext } from '@onlook/models/chat';
 import { Icons } from '@onlook/ui/icons/index';
 import React from 'react';
 import NodeIcon from '../../../LayersPanel/Tree/NodeIcon';
+import { assertNever } from '/common/helpers';
 
 export function getTruncatedName(context: ChatMessageContext) {
     let name = context.displayName;
@@ -18,19 +19,24 @@ export function getTruncatedName(context: ChatMessageContext) {
 export function getContextIcon(context: ChatMessageContext) {
     let icon: React.ComponentType | React.ReactElement | null = null;
     switch (context.type) {
-        case 'file':
+        case MessageContextType.FILE:
             icon = Icons.File;
             break;
-        case 'image':
+        case MessageContextType.IMAGE:
             icon = Icons.Image;
             break;
-        case 'error':
+        case MessageContextType.ERROR:
             icon = Icons.InfoCircled;
             break;
-        case 'highlight':
+        case MessageContextType.HIGHLIGHT:
             return (
                 <NodeIcon tagName={context.displayName} iconClass="w-3 h-3 ml-1 mr-2 flex-none" />
             );
+        case MessageContextType.PROJECT:
+            icon = Icons.Cube;
+            break;
+        default:
+            assertNever(context);
     }
     if (icon) {
         return React.createElement(icon);

packages/ai/src/prompt/base.ts

@@ -1,6 +1,6 @@
 import { PROJECT_ROOT_SIGNATURE } from './signatures';
 
-const reactRole = `Act as an expert React and Tailwind developer. Your goal is to analyze the provided code, understand the requested modifications, and implement them accurately while explaining your thought process.`;
+const reactRole = `You are running in Onlook to help users develop their app. Act as an expert React, Next.js and Tailwind developer. Your goal is to analyze the provided code, understand the requested modifications, and implement them accurately while explaining your thought process.`;
 
 const lazy = `You are diligent and tireless! You NEVER leave comments describing code without implementing it! You always COMPLETELY IMPLEMENT the needed code!`;
 

packages/ai/src/prompt/onlook.ts

@@ -0,0 +1,19 @@
+export const ONLOOK_PROMPT = `You are currently running Onlook, it is an electron app that allows users to run their own React Next.js app on their machine, chat, style and deploy it. 
+The available project folder is where the user's Next.js app lives. This is the app they are running through Onlook. 
+
+Onlook has a few features users could use:
+
+# Chat panel
+To chat with an element, select it through the UI. Let the users know what you can do through the chat whether it's with tools or otherwise.
+
+# Style editor
+There is a figma-like design panel on the right side of the app where they can change any styles. Let the users know they can also right click to get more actions.
+
+# Bottom tool bar
+There is a bottom toolbar with controls users can use for things like adding new elements, starting, and stopping their project, and even open up the terminal. 
+
+# Publish
+Once done, the user can also publish their project on the top right corner of the app. Either to a preview link or to a custom domain if they own one
+
+# Pro plan
+Upgrading to the Pro plan gives unlimited messages, more than 1 custom domain, among other perks.`;

packages/ai/src/tools/index.ts

@@ -1,6 +1,7 @@
 import { anthropic } from '@ai-sdk/anthropic';
 import { tool, type ToolSet } from 'ai';
 import { readFile } from 'fs/promises';
+import { ONLOOK_PROMPT } from 'src/prompt/onlook';
 import { z } from 'zod';
 import { getAllFiles } from './helpers';
 
@@ -22,21 +23,34 @@ export const listFilesTool = tool({
     },
 });
 
-export const readFileTool = tool({
-    description: 'Read the contents of a file',
+export const readFilesTool = tool({
+    description: 'Read the contents of files',
     parameters: z.object({
-        path: z.string().describe('The absolute path to the file to read'),
+        paths: z.array(z.string()).describe('The absolute paths to the files to read'),
     }),
-    execute: async ({ path }) => {
+    execute: async ({ paths }) => {
         try {
-            const file = await readFile(path, 'utf8');
-            return file;
+            const files = await Promise.all(
+                paths.map(async (path) => {
+                    const file = await readFile(path, 'utf8');
+                    return { path, content: file };
+                }),
+            );
+            return files;
         } catch (error) {
             return `Error: ${error instanceof Error ? error.message : error}`;
         }
     },
 });
 
+export const onlookInstructionsTool = tool({
+    description: 'Get the instructions for the Onlook AI',
+    parameters: z.object({}),
+    execute: async () => {
+        return ONLOOK_PROMPT;
+    },
+});
+
 // https://docs.anthropic.com/en/docs/agents-and-tools/computer-use#understand-anthropic-defined-tools
 // https://sdk.vercel.ai/docs/guides/computer-use#get-started-with-computer-use
 
@@ -118,5 +132,6 @@ export const getStrReplaceEditorTool = (handlers: FileOperationHandlers) => {
 
 export const chatToolSet: ToolSet = {
     list_files: listFilesTool,
-    read_file: readFileTool,
+    read_files: readFilesTool,
+    onlook_instructions: onlookInstructionsTool,
 };