Background effect

text_to_segment/.sieveignore

@@ -0,0 +1,2 @@
+outputs/*
+thumbnails/*

text_to_segment/DEV_README.md

@@ -0,0 +1,5 @@
+
+## Deploying
+Run `python scripts/make_thumbnails.py` once before using `sieve.deploy` so that the thumbnails exist.
+
+Run `python scripts/run_examples.py` to launch examples. It prints a link to each job.

text_to_segment/README.md

@@ -2,6 +2,15 @@
 # SAM2 text-to-segment
 Simple demo app to enable text prompts for SAM2.
 
+Have a look at these sieve functions to see how we use this building block!
+- [focus effect](https://www.sievedata.com/functions/sieve-internal/sam2-focus)
+- [callout effect](https://www.sievedata.com/functions/sieve-internal/sam2-callout)
+- [color filter](https://www.sievedata.com/functions/sieve-internal/sam2-color-filter)
+- [background blur](https://www.sievedata.com/functions/sieve-internal/sam2-blur)
+- [selective color](https://www.sievedata.com/functions/sieve-internal/sam2-selective-color)
+- [censorship](https://www.sievedata.com/functions/sieve-internal/sam2-pixelate)
+
+
 ## Usage
 Upload a video or a photo and name the object you want to track.
 

text_to_segment/bbox.py

@@ -0,0 +1,27 @@
+import os
+import sieve
+
+import config
+
+
+def get_object_bbox(image: sieve.File, object_name: str):
+    if config.CACHE and os.path.exists("bbox.txt"):
+        with open("bbox.txt", "r") as f:
+            return list(map(int, f.read().split(',')))
+
+    yolo = sieve.function.get('sieve/yolov8')
+
+    response = yolo.run(
+        file=image,
+        classes=object_name,
+        models='yolov8l-world',
+    )
+
+    box = response['boxes'][0]
+    bounding_box = [box['x1'],box['y1'],box['x2'],box['y2']]
+
+    if config.CACHE:
+        with open("bbox.txt", "w") as f:
+            f.write(','.join(map(str, bounding_box)))
+
+    return bounding_box

text_to_segment/blending.py

@@ -0,0 +1,56 @@
+import sieve
+import cv2
+import numpy as np
+
+from utils import resize_and_crop
+
+
+def blend_to_background(object_video: sieve.File, mask_video: sieve.File, background_img: sieve.File):
+    """
+    superimpose `object_video` onto `background_img` using `mask_video`
+
+    assumes that `mask_video` frames correspond 1-1 with `object_video` frames
+    (but framerate doesn't matter)
+    """
+
+    object_video = cv2.VideoCapture(object_video.path)
+    mask_video = cv2.VideoCapture(mask_video.path)
+    background = cv2.imread(background_img.path)
+
+    output_path = "blended_output.mp4"
+
+    frame_width = int(object_video.get(cv2.CAP_PROP_FRAME_WIDTH))
+    frame_height = int(object_video.get(cv2.CAP_PROP_FRAME_HEIGHT))
+    fps = object_video.get(cv2.CAP_PROP_FPS)
+
+    background = resize_and_crop(background, frame_width, frame_height)
+
+    fourcc = cv2.VideoWriter_fourcc(*'mp4v')
+    output_video = cv2.VideoWriter(output_path, fourcc, fps, (frame_width, frame_height))
+
+    while True:
+        ret_obj, obj_frame = object_video.read()
+        ret_mask, mask_frame = mask_video.read()
+
+        if not ret_obj or not ret_mask:
+            break
+
+        if len(mask_frame.shape) == 3:
+            mask_frame = cv2.cvtColor(mask_frame, cv2.COLOR_BGR2GRAY)
+
+        mask = mask_frame.astype(np.float32) / 255.0
+        mask = np.expand_dims(mask, axis=2)
+
+        blended_frame = (obj_frame * mask + background * (1 - mask)).astype(np.uint8)
+
+        output_video.write(blended_frame)
+
+    object_video.release()
+    mask_video.release()
+    output_video.release()
+
+    return sieve.File(path=output_path)
+
+
+
+

text_to_segment/config.py

@@ -0,0 +1 @@
+CACHE = False