diff --git a/packages/fluent-ai/package.json b/packages/fluent-ai/package.json index bd19caa..02a3299 100644 --- a/packages/fluent-ai/package.json +++ b/packages/fluent-ai/package.json @@ -49,4 +49,4 @@ "prettier": { "trailingComma": "all" } -} \ No newline at end of file +} diff --git a/packages/fluent-ai/src/builder/image.ts b/packages/fluent-ai/src/builder/image.ts index 4ef183f..8b364c0 100644 --- a/packages/fluent-ai/src/builder/image.ts +++ b/packages/fluent-ai/src/builder/image.ts @@ -40,6 +40,16 @@ export class ImageBuilder { return this; } + edit(images: string[]) { + this.input.edit = images; + return this; + } + + upload(options: ImageJob["input"]["upload"]) { + this.input.upload = options; + return this; + } + build() { return { type: "image" as const, diff --git a/packages/fluent-ai/src/job/fal.ts b/packages/fluent-ai/src/job/fal.ts index db5e3b7..2190e5f 100644 --- a/packages/fluent-ai/src/job/fal.ts +++ b/packages/fluent-ai/src/job/fal.ts @@ -1,5 +1,9 @@ import type { ImageJob } from "~/src/job/schema"; -import { createHTTPJob, downloadImages } from "~/src/job/http"; +import { + createHTTPJob, + downloadImages, + uploadLocalImages, +} from "~/src/job/http"; // TODO: switch to fal queue api const BASE_URL = "https://fal.run"; @@ -8,17 +12,28 @@ export const runner = { image: async (input: ImageJob["input"], options?: ImageJob["options"]) => { const apiKey = options?.apiKey || process.env.FAL_API_KEY; + let editImages = input.edit; + if (editImages && input.upload) { + editImages = await uploadLocalImages(editImages, input.upload); + } + + const body: any = { + prompt: input.prompt, + image_size: input.size, + num_images: input.n, + }; + + if (editImages && editImages.length > 0) { + body.image_urls = editImages; + } + const request = new Request(`${BASE_URL}/${input.model}`, { method: "POST", headers: { Authorization: `Key ${apiKey}`, "Content-Type": "application/json", }, - body: JSON.stringify({ - prompt: input.prompt, - image_size: input.size, - num_images: input.n, - }), + body: JSON.stringify(body), }); return createHTTPJob(request, async (response: Response) => { diff --git a/packages/fluent-ai/src/job/http.ts b/packages/fluent-ai/src/job/http.ts index 3f1e8f4..d6a5f61 100644 --- a/packages/fluent-ai/src/job/http.ts +++ b/packages/fluent-ai/src/job/http.ts @@ -1,6 +1,54 @@ -import * as fs from "node:fs"; +import * as fs from "node:fs/promises"; import * as path from "node:path"; -import type { ImageJob } from "./schema"; +import type { ImageJob } from "~/src/job/schema"; + +const MIME_TYPES: Record = { + ".png": "image/png", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".gif": "image/gif", + ".webp": "image/webp", +}; + +function isRemoteUrl(url: string) { + return url.startsWith("http://") || url.startsWith("https://"); +} + +async function convertToBase64(filePath: string) { + try { + const fileBuffer = await fs.readFile(filePath); + const base64Data = fileBuffer.toString("base64"); + + const ext = path.extname(filePath).toLowerCase(); + const mimeType = MIME_TYPES[ext] || "image/png"; + + return `data:${mimeType};base64,${base64Data}`; + } catch (error) { + console.error(`Failed to read local file ${filePath}:`, error); + throw new Error(`Failed to upload local image: ${filePath}`); + } +} + +export async function uploadLocalImages( + images: string[], + uploadOption: ImageJob["input"]["upload"], +) { + if (!uploadOption) { + return images; + } + + return Promise.all( + images.map(async (img) => { + if (isRemoteUrl(img)) return img; + + if (uploadOption === "base64") { + return await convertToBase64(img); + } + + return img; + }), + ); +} export async function createHTTPJob( request: RequestInfo | URL, @@ -24,8 +72,10 @@ export async function downloadImages( ): Promise> { const localDir = options!.local; - if (!fs.existsSync(localDir)) { - fs.mkdirSync(localDir, { recursive: true }); + try { + await fs.access(localDir); + } catch { + await fs.mkdir(localDir, { recursive: true }); } const downloadedImages = await Promise.all( @@ -47,7 +97,7 @@ export async function downloadImages( const filePath = path.join(localDir, filename); const buffer = await response.arrayBuffer(); - fs.writeFileSync(filePath, Buffer.from(buffer)); + await fs.writeFile(filePath, Buffer.from(buffer)); return { ...img, diff --git a/packages/fluent-ai/src/job/schema.ts b/packages/fluent-ai/src/job/schema.ts index 3a9338a..8abe250 100644 --- a/packages/fluent-ai/src/job/schema.ts +++ b/packages/fluent-ai/src/job/schema.ts @@ -53,6 +53,7 @@ const embeddingOutputSchema = z.object({ }); const downloadOptionsSchema = z.union([z.object({ local: z.string() })]); +const uploadOptionsSchema = z.union([z.literal("base64")]); const imageSizeSchema = z.object({ width: z.number(), @@ -69,6 +70,8 @@ const imageInputSchema = z.object({ outputFormat: z.string().optional(), guidanceScale: z.number().optional(), download: downloadOptionsSchema.optional(), + edit: z.array(z.string()).optional(), + upload: uploadOptionsSchema.optional(), }); const imageOutputSchema = z.object({ diff --git a/packages/fluent-ai/test/job.test.ts b/packages/fluent-ai/test/job.test.ts index 8400913..fb58247 100644 --- a/packages/fluent-ai/test/job.test.ts +++ b/packages/fluent-ai/test/job.test.ts @@ -7,6 +7,7 @@ import { type Job, user, voyage, + type ImageJob, } from "~/src/index"; import { Runner } from "~/src/job/runner"; @@ -55,6 +56,42 @@ test("image job", () => { ); }); +test("image edit job", () => { + const job: ImageJob = { + provider: "fal", + type: "image", + input: { + model: "fal-ai/bytedance/seedream/v4/edit", + prompt: + "Dress the model in the clothes and hat. Add a cat to the scene and change the background to a Victorian era building.", + edit: [ + "./input1.png", + "https://storage.googleapis.com/falserverless/example_inputs/seedream4_edit_input_2.png", + ], + size: { + width: 3840, + height: 2160, + }, + upload: "base64", + }, + }; + + expect(job).toEqual( + fal() + .image("fal-ai/bytedance/seedream/v4/edit") + .prompt( + "Dress the model in the clothes and hat. Add a cat to the scene and change the background to a Victorian era building.", + ) + .edit([ + "./input1.png", + "https://storage.googleapis.com/falserverless/example_inputs/seedream4_edit_input_2.png", + ]) + .size({ width: 3840, height: 2160 }) + .upload("base64") + .build(), + ); +}); + test("embedding job", () => { const job: Job = { provider: "voyage", diff --git a/packages/playground/app/components/image.tsx b/packages/playground/app/components/image.tsx index 8cfb726..ac58313 100644 --- a/packages/playground/app/components/image.tsx +++ b/packages/playground/app/components/image.tsx @@ -11,6 +11,16 @@ import { import { Textarea } from "~/components/ui/textarea"; import { ScrollArea } from "~/components/ui/scroll-area"; import type { ImageJob } from "fluent-ai"; +import { useRef } from "react"; +import { + Paperclip, + Loader2, + Palette, + Image as ImageIcon, + X, + Download, + ExternalLink, +} from "lucide-react"; interface Provider { name: string; @@ -18,7 +28,7 @@ interface Provider { models: Array<{ id: string; name: string }>; } -interface ImageProps { +interface ImagePlaygroundProps { job: ImageJob; onChange: (job: ImageJob) => void; providers: Provider[]; @@ -36,7 +46,9 @@ export const ImagePlayground = ({ loading = false, error = null, output = null, -}: ImageProps) => { +}: ImagePlaygroundProps) => { + const fileInputRef = useRef(null); + const handleSubmit = (e: React.FormEvent) => { e.preventDefault(); onSubmit(job); @@ -82,6 +94,37 @@ export const ImagePlayground = ({ }); } + function setEditImages(files: FileList | null) { + if (!files || files.length === 0) return; + + const imagePaths = Array.from(files).map((file) => { + return URL.createObjectURL(file); + }); + + const existingImages = input.edit || []; + + onChange({ + ...job, + input: { + ...input, + edit: [...existingImages, ...imagePaths], + }, + }); + } + + function removeEditImage(index: number) { + const newEdit = [...(input.edit || [])]; + URL.revokeObjectURL(newEdit[index]); + newEdit.splice(index, 1); + onChange({ + ...job, + input: { + ...input, + edit: newEdit.length > 0 ? newEdit : undefined, + }, + }); + } + function setSize(sizeStr: string) { const [width, height] = sizeStr.split("x").map(Number); onChange({ @@ -258,6 +301,75 @@ export const ImagePlayground = ({ Describe the image you want to generate

+ +
+ + { + setEditImages(e.target.files); + e.target.value = ""; + }} + className="hidden" + /> + +

+ Upload images to edit or modify (supports multiple files) +

+ {input.edit && input.edit.length > 0 && ( +
+

+ {input.edit.length} image + {input.edit.length > 1 ? "s" : ""} uploaded +

+
+ {input.edit.map((imagePath, index) => ( +
+
+ {`Edit + +
+
+ + Image {index + 1} + +
+
+ ))} +
+
+ )} +
@@ -271,12 +383,12 @@ export const ImagePlayground = ({ > {loading ? ( <> - + Generating... ) : ( <> - 🎨 + Generate Images )} @@ -294,7 +406,7 @@ export const ImagePlayground = ({ {loading && (
-
🎨
+

Generating your images...

@@ -360,6 +472,7 @@ export const ImagePlayground = ({ link.click(); }} > + Download
@@ -397,7 +511,7 @@ export const ImagePlayground = ({ {!output && !error && !loading && (
-
🖼️
+

No images generated yet.

Enter a prompt and click "Generate Images" to get started.