diff --git a/packages/fluent-ai/package.json b/packages/fluent-ai/package.json
index bd19caa..02a3299 100644
--- a/packages/fluent-ai/package.json
+++ b/packages/fluent-ai/package.json
@@ -49,4 +49,4 @@
"prettier": {
"trailingComma": "all"
}
-}
\ No newline at end of file
+}
diff --git a/packages/fluent-ai/src/builder/image.ts b/packages/fluent-ai/src/builder/image.ts
index 4ef183f..8b364c0 100644
--- a/packages/fluent-ai/src/builder/image.ts
+++ b/packages/fluent-ai/src/builder/image.ts
@@ -40,6 +40,16 @@ export class ImageBuilder {
return this;
}
+ edit(images: string[]) {
+ this.input.edit = images;
+ return this;
+ }
+
+ upload(options: ImageJob["input"]["upload"]) {
+ this.input.upload = options;
+ return this;
+ }
+
build() {
return {
type: "image" as const,
diff --git a/packages/fluent-ai/src/job/fal.ts b/packages/fluent-ai/src/job/fal.ts
index db5e3b7..2190e5f 100644
--- a/packages/fluent-ai/src/job/fal.ts
+++ b/packages/fluent-ai/src/job/fal.ts
@@ -1,5 +1,9 @@
import type { ImageJob } from "~/src/job/schema";
-import { createHTTPJob, downloadImages } from "~/src/job/http";
+import {
+ createHTTPJob,
+ downloadImages,
+ uploadLocalImages,
+} from "~/src/job/http";
// TODO: switch to fal queue api
const BASE_URL = "https://fal.run";
@@ -8,17 +12,28 @@ export const runner = {
image: async (input: ImageJob["input"], options?: ImageJob["options"]) => {
const apiKey = options?.apiKey || process.env.FAL_API_KEY;
+ let editImages = input.edit;
+ if (editImages && input.upload) {
+ editImages = await uploadLocalImages(editImages, input.upload);
+ }
+
+ const body: any = {
+ prompt: input.prompt,
+ image_size: input.size,
+ num_images: input.n,
+ };
+
+ if (editImages && editImages.length > 0) {
+ body.image_urls = editImages;
+ }
+
const request = new Request(`${BASE_URL}/${input.model}`, {
method: "POST",
headers: {
Authorization: `Key ${apiKey}`,
"Content-Type": "application/json",
},
- body: JSON.stringify({
- prompt: input.prompt,
- image_size: input.size,
- num_images: input.n,
- }),
+ body: JSON.stringify(body),
});
return createHTTPJob(request, async (response: Response) => {
diff --git a/packages/fluent-ai/src/job/http.ts b/packages/fluent-ai/src/job/http.ts
index 3f1e8f4..d6a5f61 100644
--- a/packages/fluent-ai/src/job/http.ts
+++ b/packages/fluent-ai/src/job/http.ts
@@ -1,6 +1,54 @@
-import * as fs from "node:fs";
+import * as fs from "node:fs/promises";
import * as path from "node:path";
-import type { ImageJob } from "./schema";
+import type { ImageJob } from "~/src/job/schema";
+
+const MIME_TYPES: Record = {
+ ".png": "image/png",
+ ".jpg": "image/jpeg",
+ ".jpeg": "image/jpeg",
+ ".gif": "image/gif",
+ ".webp": "image/webp",
+};
+
+function isRemoteUrl(url: string) {
+ return url.startsWith("http://") || url.startsWith("https://");
+}
+
+async function convertToBase64(filePath: string) {
+ try {
+ const fileBuffer = await fs.readFile(filePath);
+ const base64Data = fileBuffer.toString("base64");
+
+ const ext = path.extname(filePath).toLowerCase();
+ const mimeType = MIME_TYPES[ext] || "image/png";
+
+ return `data:${mimeType};base64,${base64Data}`;
+ } catch (error) {
+ console.error(`Failed to read local file ${filePath}:`, error);
+ throw new Error(`Failed to upload local image: ${filePath}`);
+ }
+}
+
+export async function uploadLocalImages(
+ images: string[],
+ uploadOption: ImageJob["input"]["upload"],
+) {
+ if (!uploadOption) {
+ return images;
+ }
+
+ return Promise.all(
+ images.map(async (img) => {
+ if (isRemoteUrl(img)) return img;
+
+ if (uploadOption === "base64") {
+ return await convertToBase64(img);
+ }
+
+ return img;
+ }),
+ );
+}
export async function createHTTPJob(
request: RequestInfo | URL,
@@ -24,8 +72,10 @@ export async function downloadImages(
): Promise> {
const localDir = options!.local;
- if (!fs.existsSync(localDir)) {
- fs.mkdirSync(localDir, { recursive: true });
+ try {
+ await fs.access(localDir);
+ } catch {
+ await fs.mkdir(localDir, { recursive: true });
}
const downloadedImages = await Promise.all(
@@ -47,7 +97,7 @@ export async function downloadImages(
const filePath = path.join(localDir, filename);
const buffer = await response.arrayBuffer();
- fs.writeFileSync(filePath, Buffer.from(buffer));
+ await fs.writeFile(filePath, Buffer.from(buffer));
return {
...img,
diff --git a/packages/fluent-ai/src/job/schema.ts b/packages/fluent-ai/src/job/schema.ts
index 3a9338a..8abe250 100644
--- a/packages/fluent-ai/src/job/schema.ts
+++ b/packages/fluent-ai/src/job/schema.ts
@@ -53,6 +53,7 @@ const embeddingOutputSchema = z.object({
});
const downloadOptionsSchema = z.union([z.object({ local: z.string() })]);
+const uploadOptionsSchema = z.union([z.literal("base64")]);
const imageSizeSchema = z.object({
width: z.number(),
@@ -69,6 +70,8 @@ const imageInputSchema = z.object({
outputFormat: z.string().optional(),
guidanceScale: z.number().optional(),
download: downloadOptionsSchema.optional(),
+ edit: z.array(z.string()).optional(),
+ upload: uploadOptionsSchema.optional(),
});
const imageOutputSchema = z.object({
diff --git a/packages/fluent-ai/test/job.test.ts b/packages/fluent-ai/test/job.test.ts
index 8400913..fb58247 100644
--- a/packages/fluent-ai/test/job.test.ts
+++ b/packages/fluent-ai/test/job.test.ts
@@ -7,6 +7,7 @@ import {
type Job,
user,
voyage,
+ type ImageJob,
} from "~/src/index";
import { Runner } from "~/src/job/runner";
@@ -55,6 +56,42 @@ test("image job", () => {
);
});
+test("image edit job", () => {
+ const job: ImageJob = {
+ provider: "fal",
+ type: "image",
+ input: {
+ model: "fal-ai/bytedance/seedream/v4/edit",
+ prompt:
+ "Dress the model in the clothes and hat. Add a cat to the scene and change the background to a Victorian era building.",
+ edit: [
+ "./input1.png",
+ "https://storage.googleapis.com/falserverless/example_inputs/seedream4_edit_input_2.png",
+ ],
+ size: {
+ width: 3840,
+ height: 2160,
+ },
+ upload: "base64",
+ },
+ };
+
+ expect(job).toEqual(
+ fal()
+ .image("fal-ai/bytedance/seedream/v4/edit")
+ .prompt(
+ "Dress the model in the clothes and hat. Add a cat to the scene and change the background to a Victorian era building.",
+ )
+ .edit([
+ "./input1.png",
+ "https://storage.googleapis.com/falserverless/example_inputs/seedream4_edit_input_2.png",
+ ])
+ .size({ width: 3840, height: 2160 })
+ .upload("base64")
+ .build(),
+ );
+});
+
test("embedding job", () => {
const job: Job = {
provider: "voyage",
diff --git a/packages/playground/app/components/image.tsx b/packages/playground/app/components/image.tsx
index 8cfb726..ac58313 100644
--- a/packages/playground/app/components/image.tsx
+++ b/packages/playground/app/components/image.tsx
@@ -11,6 +11,16 @@ import {
import { Textarea } from "~/components/ui/textarea";
import { ScrollArea } from "~/components/ui/scroll-area";
import type { ImageJob } from "fluent-ai";
+import { useRef } from "react";
+import {
+ Paperclip,
+ Loader2,
+ Palette,
+ Image as ImageIcon,
+ X,
+ Download,
+ ExternalLink,
+} from "lucide-react";
interface Provider {
name: string;
@@ -18,7 +28,7 @@ interface Provider {
models: Array<{ id: string; name: string }>;
}
-interface ImageProps {
+interface ImagePlaygroundProps {
job: ImageJob;
onChange: (job: ImageJob) => void;
providers: Provider[];
@@ -36,7 +46,9 @@ export const ImagePlayground = ({
loading = false,
error = null,
output = null,
-}: ImageProps) => {
+}: ImagePlaygroundProps) => {
+ const fileInputRef = useRef(null);
+
const handleSubmit = (e: React.FormEvent) => {
e.preventDefault();
onSubmit(job);
@@ -82,6 +94,37 @@ export const ImagePlayground = ({
});
}
+ function setEditImages(files: FileList | null) {
+ if (!files || files.length === 0) return;
+
+ const imagePaths = Array.from(files).map((file) => {
+ return URL.createObjectURL(file);
+ });
+
+ const existingImages = input.edit || [];
+
+ onChange({
+ ...job,
+ input: {
+ ...input,
+ edit: [...existingImages, ...imagePaths],
+ },
+ });
+ }
+
+ function removeEditImage(index: number) {
+ const newEdit = [...(input.edit || [])];
+ URL.revokeObjectURL(newEdit[index]);
+ newEdit.splice(index, 1);
+ onChange({
+ ...job,
+ input: {
+ ...input,
+ edit: newEdit.length > 0 ? newEdit : undefined,
+ },
+ });
+ }
+
function setSize(sizeStr: string) {
const [width, height] = sizeStr.split("x").map(Number);
onChange({
@@ -258,6 +301,75 @@ export const ImagePlayground = ({
Describe the image you want to generate
+
+
+
+
{
+ setEditImages(e.target.files);
+ e.target.value = "";
+ }}
+ className="hidden"
+ />
+
+
+ Upload images to edit or modify (supports multiple files)
+
+ {input.edit && input.edit.length > 0 && (
+
+
+ {input.edit.length} image
+ {input.edit.length > 1 ? "s" : ""} uploaded
+
+
+ {input.edit.map((imagePath, index) => (
+
+
+

+
+
+
+
+ Image {index + 1}
+
+
+
+ ))}
+
+
+ )}
+
@@ -271,12 +383,12 @@ export const ImagePlayground = ({
>
{loading ? (
<>
- ⏳
+
Generating...
>
) : (
<>
- 🎨
+
Generate Images
>
)}
@@ -294,7 +406,7 @@ export const ImagePlayground = ({
{loading && (
-
🎨
+
Generating your images...
@@ -360,6 +472,7 @@ export const ImagePlayground = ({
link.click();
}}
>
+
Download
@@ -397,7 +511,7 @@ export const ImagePlayground = ({
{!output && !error && !loading && (
-
🖼️
+
No images generated yet.
Enter a prompt and click "Generate Images" to get started.