mattyamonaca/pbremtools

Precise background remover

pbremtools's Issues

Error when using SAM_CLIP

Traceback (most recent call last):
File "A:\v1.5\stable-diffusion-webui\venv\lib\site-packages\gradio\routes.py", line 394, in run_predict
output = await app.get_blocks().process_api(
File "A:\v1.5\stable-diffusion-webui\venv\lib\site-packages\gradio\blocks.py", line 1075, in process_api
result = await self.call_function(
File "A:\v1.5\stable-diffusion-webui\venv\lib\site-packages\gradio\blocks.py", line 884, in call_function
prediction = await anyio.to_thread.run_sync(
File "A:\v1.5\stable-diffusion-webui\venv\lib\site-packages\anyio\to_thread.py", line 31, in run_sync
return await get_asynclib().run_sync_in_worker_thread(
File "A:\v1.5\stable-diffusion-webui\venv\lib\site-packages\anyio_backends_asyncio.py", line 937, in run_sync_in_worker_thread
return await future
File "A:\v1.5\stable-diffusion-webui\venv\lib\site-packages\anyio_backends_asyncio.py", line 867, in run
result = context.run(func, *args)
File "A:\v1.5\stable-diffusion-webui\extensions\PBRemTools\scripts\main.py", line 19, in processing
mask, image = get_foreground(image, td_abg_enabled, h_split, v_split, n_cluster, alpha, th_rate, cascadePSP_enabled, fast, psp_L, sa_enabled, seg_query)
File "A:\v1.5\stable-diffusion-webui\extensions\PBRemTools\scripts\td_abg.py", line 131, in get_foreground
mask = get_sa_mask(img, query)
File "A:\v1.5\stable-diffusion-webui\extensions\PBRemTools\scripts\sa_mask.py", line 147, in get_sa_mask
masks = segment(predicted_iou_threshold, stability_score_threshold, clip_threshold, image, query)
File "A:\v1.5\stable-diffusion-webui\extensions\PBRemTools\scripts\sa_mask.py", line 126, in segment
mask_generator = load_mask_generator()
File "A:\v1.5\stable-diffusion-webui\extensions\PBRemTools\scripts\sa_mask.py", line 29, in load_mask_generator
sam = sam_model_registryMODEL_TYPE.to(device)
File "A:\v1.5\stable-diffusion-webui\venv\lib\site-packages\segment_anything\build_sam.py", line 15, in build_sam_vit_h
return _build_sam(
File "A:\v1.5\stable-diffusion-webui\venv\lib\site-packages\segment_anything\build_sam.py", line 105, in _build_sam
state_dict = torch.load(f)
File "A:\v1.5\stable-diffusion-webui\modules\safe.py", line 106, in load
return load_with_extra(filename, extra_handler=global_extra_handler, *args, **kwargs)
File "A:\v1.5\stable-diffusion-webui\modules\safe.py", line 151, in load_with_extra
return unsafe_torch_load(filename, *args, **kwargs)
File "A:\v1.5\stable-diffusion-webui\venv\lib\site-packages\torch\serialization.py", line 815, in load
return _legacy_load(opened_file, map_location, pickle_module, **pickle_load_args)
File "A:\v1.5\stable-diffusion-webui\venv\lib\site-packages\torch\serialization.py", line 1033, in _legacy_load
magic_number = pickle_module.load(f, **pickle_load_args)
_pickle.UnpicklingError: A load persistent id instruction was encountered,
but no persistent_load function was specified.

[feature request]load the model from the model folder

can we load sam model form models folder not extensions?

other extension both use sam model and it is big for 2.5G

such as this extension
continue-revolution/sd-webui-segment-anything#20

No module named 'modules.paths_internal'

Error prompt：No module named 'modules.paths_internal'

can't extract prompt object but only remove background

I have install all need packages and have not error in console, but no matter what prompt i add, it only remove background. The tile division BG Removers and cascadePSP I set to disable.

Doesn't work

*** Error loading script: api.py
Traceback (most recent call last):
File "D:\stable-diffusion-portable-main\modules\scripts.py", line 319, in load_scripts
script_module = script_loading.load_module(scriptfile.path)
File "D:\stable-diffusion-portable-main\modules\script_loading.py", line 10, in load_module
module_spec.loader.exec_module(module)
File "", line 883, in exec_module
File "", line 241, in call_with_frames_removed
File "D:\stable-diffusion-portable-main\extensions\PBRemTools\scripts\api.py", line 9, in
from scripts.tools import process_image, model_list
File "D:\stable-diffusion-portable-main\extensions\PBRemTools\scripts\tools.py", line 11, in
from scripts.td_abg import get_foreground
File "D:\stable-diffusion-portable-main\extensions\PBRemTools\scripts\td_abg.py", line 5, in
from sklearn.cluster import KMeans, MiniBatchKMeans
File "D:\stable-diffusion-portable-main\venv\lib\site-packages\sklearn_init.py", line 83, in
from .base import clone
File "D:\stable-diffusion-portable-main\venv\lib\site-packages\sklearn\base.py", line 19, in
from .utils import _IS_32BIT
ModuleNotFoundError: No module named 'sklearn.utils'

---

*** Error loading script: main.py
Traceback (most recent call last):
File "D:\stable-diffusion-portable-main\modules\scripts.py", line 319, in load_scripts
script_module = script_loading.load_module(scriptfile.path)
File "D:\stable-diffusion-portable-main\modules\script_loading.py", line 10, in load_module
module_spec.loader.exec_module(module)
File "", line 883, in exec_module
File "", line 241, in call_with_frames_removed
File "D:\stable-diffusion-portable-main\extensions\PBRemTools\scripts\main.py", line 2, in
from scripts.tools import processing, model_list
File "D:\stable-diffusion-portable-main\extensions\PBRemTools\scripts\tools.py", line 11, in
from scripts.td_abg import get_foreground
File "D:\stable-diffusion-portable-main\extensions\PBRemTools\scripts\td_abg.py", line 5, in
from sklearn.cluster import KMeans, MiniBatchKMeans
File "D:\stable-diffusion-portable-main\venv\lib\site-packages\sklearn_init.py", line 83, in
from .base import clone
File "D:\stable-diffusion-portable-main\venv\lib\site-packages\sklearn\base.py", line 19, in
from .utils import _IS_32BIT
ModuleNotFoundError: No module named 'sklearn.utils'

---

*** Error loading script: td_abg.py
Traceback (most recent call last):
File "D:\stable-diffusion-portable-main\modules\scripts.py", line 319, in load_scripts
script_module = script_loading.load_module(scriptfile.path)
File "D:\stable-diffusion-portable-main\modules\script_loading.py", line 10, in load_module
module_spec.loader.exec_module(module)
File "", line 883, in exec_module
File "", line 241, in call_with_frames_removed
File "D:\stable-diffusion-portable-main\extensions\PBRemTools\scripts\td_abg.py", line 5, in
from sklearn.cluster import KMeans, MiniBatchKMeans
File "D:\stable-diffusion-portable-main\venv\lib\site-packages\sklearn_init.py", line 83, in
from .base import clone
File "D:\stable-diffusion-portable-main\venv\lib\site-packages\sklearn\base.py", line 19, in
from .utils import _IS_32BIT
ModuleNotFoundError: No module named 'sklearn.utils'

---

*** Error loading script: tools.py
Traceback (most recent call last):
File "D:\stable-diffusion-portable-main\modules\scripts.py", line 319, in load_scripts
script_module = script_loading.load_module(scriptfile.path)
File "D:\stable-diffusion-portable-main\modules\script_loading.py", line 10, in load_module
module_spec.loader.exec_module(module)
File "", line 883, in exec_module
File "", line 241, in call_with_frames_removed
File "D:\stable-diffusion-portable-main\extensions\PBRemTools\scripts\tools.py", line 11, in
from scripts.td_abg import get_foreground
File "D:\stable-diffusion-portable-main\extensions\PBRemTools\scripts\td_abg.py", line 5, in
from sklearn.cluster import KMeans, MiniBatchKMeans
File "D:\stable-diffusion-portable-main\venv\lib\site-packages\sklearn_init.py", line 83, in
from .base import clone
File "D:\stable-diffusion-portable-main\venv\lib\site-packages\sklearn\base.py", line 19, in
from .utils import _IS_32BIT
ModuleNotFoundError: No module named `'sklearn.utils'

{'error': 'OSError', 'detail': '', 'body': '', 'errors': 'cannot write mode RGBA as JPEG'}

code:

def img_to_base64(img_path):
    img = Image.open(img_path)
    rgb_img = img.convert('RGB')
    buffered = io.BytesIO()
    rgb_img.save(buffered, format="JPEG")
    img_str = base64.b64encode(buffered.getvalue())
    img_base64_str = img_str.decode()
    return img_base64_str
img_base64 = img_to_base64(output_file)  # replace with your image path
payload_SAM["img"] = img_base64
response = requests.post(url=f'{url}/pbrem/predict', json=payload_SAM)

HELP!!! Module Not FoundError: No module named 'modules.paths_internal'

Closing server running on port: 7860
Restarting UI...
Error loading script: main.py
Traceback (most recent call last):
File "\modules\scripts.py", line 248, in load_scripts
script_module = script_loading.load_module(scriptfile.path)
File "\modules\script_loading.py", line 11, in load_module
module_spec.loader.exec_module(module)
File "", line 883, in exec_module
File "", line 241, in _call_with_frames_removed
File "\extensions\PBRemTools\scripts\main.py", line 12, in
from scripts.td_abg import get_foreground
File "\extensions\PBRemTools\scripts\td_abg.py", line 30, in
from scripts.sa_mask import get_sa_mask
File "*****\extensions\PBRemTools\scripts\sa_mask.py", line 17, in
from modules.paths_internal import extensions_dir
ModuleNotFoundError: No module named 'modules.paths_internal'

Error loading script: sa_mask.py
Traceback (most recent call last):
File "\modules\scripts.py", line 248, in load_scripts
script_module = script_loading.load_module(scriptfile.path)
File "\modules\script_loading.py", line 11, in load_module
module_spec.loader.exec_module(module)
File "", line 883, in exec_module
File "", line 241, in _call_with_frames_removed
File "*****\extensions\PBRemTools\scripts\sa_mask.py", line 17, in
from modules.paths_internal import extensions_dir
ModuleNotFoundError: No module named 'modules.paths_internal'

Error loading script: td_abg.py
Traceback (most recent call last):
File "\modules\scripts.py", line 248, in load_scripts
script_module = script_loading.load_module(scriptfile.path)
File "\modules\script_loading.py", line 11, in load_module
module_spec.loader.exec_module(module)
File "", line 883, in exec_module
File "", line 241, in _call_with_frames_removed
File "\extensions\PBRemTools\scripts\td_abg.py", line 30, in
from scripts.sa_mask import get_sa_mask
File "\extensions\PBRemTools\scripts\sa_mask.py", line 17, in
from modules.paths_internal import extensions_dir
ModuleNotFoundError: No module named 'modules.paths_internal'

install suggestion

this should be a pr, but I'm lazy. Here's a recommendation for changing the install script

import launch

if not launch.is_installed("onnx"):
    launch.run_pip("install onnx", "Installing onnx...")

if not launch.is_installed("onnxruntime"):
    launch.run_pip("install onnxruntime-gpu", "Installing onnxruntime-gpu...")

if not launch.is_installed("cv2"):
    launch.run_pip("install opencv-python", "Installing opencv-python...")

if not launch.is_installed("numpy"):
    launch.run_pip("install numpy", "Installing numpy...")

if not launch.is_installed("PIL"):
    launch.run_pip("install Pillow", "Installing Pillow...")

if not launch.is_installed("segmentation-refinement"):
    launch.run_pip("install segmentation-refinement", "Installing segmentation-refinement...")

if not launch.is_installed("sklearn"):
    launch.run_pip("install scikit-learn", "Installing scikit-learn...")

CUDA out of memory, Png format bug, cascadePSP doesn't work attempts to download a model to the user folder

problem 1. -> CUDA out of memory when using "Segment Anything" and "Tile division" both at the same time. Then automatic gets bugged and I need to restart.

They work ok one at the time.

Maybe I'm using this wrong, are those two options intended to be used at the same time?

problem 2. -> I've also had that error about the png format that is in another thread but changing the code inside "components.py" appear to have been fixed the issue.

problem 3. ->
cascadePSP doesn't work, it attempts to download a model inside windows user folder, not in the extension model folder. (but fails to start downloading)
error is:
MD5 of the model file does not match Downloading the model again... Downloading model file into C:\Users\USERNAME/.segmentation-refinement\model

can this be downloaded manually like the "Segment Anything" model?

Is it only useful for characters ?

This works.

This does not work.

Memory clean after generate

In cascadePSP I set the memory value too high and got a CUDA error. After that, I couldn't generate anything because the model was stuck in video memory and kept taking up video memory.
Is it possible to add a function that automatically removes the model from memory after generating? Option or button is fine, but then I wouldn't have to restart the whole webui.

Breaks rembg extension upon install, sometimes.

Traceback:

File "D:\NasD\stable-diffusion-webui\modules\call_queue.py", line 56, in f
res = list(func(*args, **kwargs))
File "D:\NasD\stable-diffusion-webui\modules\call_queue.py", line 37, in f
res = func(*args, **kwargs)
File "D:\NasD\stable-diffusion-webui\modules\postprocessing.py", line 56, in run_postprocessing
scripts.scripts_postproc.run(pp, args)
File "D:\NasD\stable-diffusion-webui\modules\scripts_postprocessing.py", line 130, in run
script.process(pp, **process_args)
File "D:\NasD\stable-diffusion-webui\extensions\stable-diffusion-webui-rembg\scripts\postprocessing_rembg.py", line 53, in process
session=rembg.new_session(model),
File "D:\NasD\stable-diffusion-webui\venv\lib\site-packages\rembg\session_factory.py", line 66, in new_session
ort.InferenceSession(
File "D:\NasD\stable-diffusion-webui\venv\lib\site-packages\onnxruntime\capi\onnxruntime_inference_collection.py", line 360, in init
self._create_inference_session(providers, provider_options, disabled_optimizers)
File "D:\NasD\stable-diffusion-webui\venv\lib\site-packages\onnxruntime\capi\onnxruntime_inference_collection.py", line 408, in _create_inference_session
sess.initialize_session(providers, provider_options, disabled_optimizers)
RuntimeError: D:\a_work\1\s\onnxruntime\core\session\provider_bridge_ort.cc:1106 onnxruntime::ProviderLibrary::Get [ONNXRuntimeError] : 1 : FAIL : LoadLibrary failed with error 126 "" when trying to load "D:\NasD\stable-diffusion-webui\venv\lib\site-packages\onnxruntime\capi\onnxruntime_providers_tensorrt.dll"

Platform: Win10pro

Attempted fix: delete \venv\lib\site-packages\onnxruntime\ and relaunch webui-user.bat, however pbremtools no longer functions afterwards.

The funny thing is, just the other day I could have sworn I was using both tools at the same time.

TypeError: slice indices must be integers or None or have an __index__ method

Some Image use SAM will raise exception.

  File "/home/abc/stable-diffusion-webui/extensions/PBRemTools/scripts/main.py", line 36, in processing
    processed = process_image(single_image, *rem_args)
  File "/home/abc/stable-diffusion-webui/extensions/PBRemTools/scripts/main.py", line 84, in process_image
    mask, image = get_foreground(image, *rem_args)
  File "/home/abc/stable-diffusion-webui/extensions/PBRemTools/scripts/td_abg.py", line 106, in get_foreground
    mask = get_sa_mask(img, query, model_name, predicted_iou_threshold, stability_score_threshold, clip_threshold)
  File "/home/abc/stable-diffusion-webui/extensions/PBRemTools/scripts/sa_mask.py", line 160, in get_sa_mask
    masks = segment(predicted_iou_threshold, stability_score_threshold, clip_threshold, image, query, model_name)
  File "/home/abc/stable-diffusion-webui/extensions/PBRemTools/scripts/sa_mask.py", line 145, in segment
    masks = filter_masks(
  File "/home/abc/stable-diffusion-webui/extensions/PBRemTools/scripts/sa_mask.py", line 105, in filter_masks
    crop = image[y : y + h, x : x + w]
TypeError: slice indices must be integers or None or have an __index__ method

I debug it and found mask["bbox"] sometimes is float so it will raise above exception.

        filtered_masks.append(mask)
        print(mask["bbox"])
        # output: [163.0, 20.0, 18.0, 31.0]
        x, y, w, h = mask["bbox"]
        crop = image[y : y + h, x : x + w]

I will open a PR for this.

Batch jpeg processing results in jpeg instead of transparent png

If you try to remove bg one by one it works alright. However uploading multiple jpeg files outputs images with black background. Uploading multiple png is also not a problem. I am not sure if this is caused by the webui fork i use but it is weird.

ModuleNotFoundError: No module named 'modules.paths_internal'

Thank you for your contribution.
I met error while restarting webui.

stable-diffusion-webui/extensions/PBRemTools/scripts/sa_mask.py", line 17, in <module>
    from modules.paths_internal import extensions_dir
ModuleNotFoundError: No module named 'modules.paths_internal'

Could you find this reason?

Is it deliberate that the header is spelt "Segment Anithing"?

The Segment Anything & CLIP portion of the UI appears to be spelt "Segment Anithing", with an i not a y. Is this deliberate?

[Feature Request] Annotated Segment Output

Can we get an 'annotation' mode where the model is used simply to annotate what it thinks different parts of the image are? I ask because sometimes the model has incorrectly labeled something I want to keep but I can't seem to identify what it thinks that object is so I can't get it to keep that part.

For example, a part of someones shirt was being ignored despite the shirt itself being correctly included in the prompt. The shirt had patterns so I wasn't sure if the mask was less than ideal, or if it thought something was in front of the shirt.

There seems to already be an example of how to do this here, but I am not familiar enough with python to add it to your extension:
https://github.com/facebookresearch/segment-anything/blob/main/notebooks/automatic_mask_generator_example.ipynb

Only needed addition to that would need to be to show what the clip labels are. If you wanted to get really fancy you might be able to make it so you could just click each segment you want to keep.

I suspect this will help a lot with people who are having trouble selecting things using the input prompt.

Getting a unexpected keyword argument 'format' error

Running on Automatic1111

Traceback (most recent call last):
  File "O:\Automatic1111\stable-diffusion-webui\venv\lib\site-packages\gradio\routes.py", line 422, in run_predict
    output = await app.get_blocks().process_api(
  File "O:\Automatic1111\stable-diffusion-webui\venv\lib\site-packages\gradio\blocks.py", line 1326, in process_api
    data = self.postprocess_data(fn_index, result["prediction"], state)
  File "O:\Automatic1111\stable-diffusion-webui\venv\lib\site-packages\gradio\blocks.py", line 1260, in postprocess_data
    prediction_value = block.postprocess(prediction_value)
  File "O:\Automatic1111\stable-diffusion-webui\venv\lib\site-packages\gradio\components.py", line 4457, in postprocess
    file = self.img_array_to_temp_file(img, dir=self.DEFAULT_TEMP_DIR)
  File "O:\Automatic1111\stable-diffusion-webui\venv\lib\site-packages\gradio\components.py", line 355, in img_array_to_temp_file
    return self.pil_to_temp_file(pil_image, dir, format="png")
TypeError: save_pil_to_file() got an unexpected keyword argument 'format'

This error appears when an image is uploaded or dragged (it doesn't matter if the image is JPG or PNG).

[Bug] Certain requirements are always attempted to be installed again at launch everytime

Because the name of some libraries are wrong, launch.is_installed() always returns False and tries to install the package again.

PBRemTools/install.py

Lines 3 to 12 in b179be6

	packages = {
	"onnx": "onnx",
	"onnxruntime-gpu": "onnxruntime-gpu==1.14.0",
	"opencv-python": "opencv-python",
	"numpy": "numpy",
	"Pillow": "Pillow",
	"segmentation-refinement": "segmentation-refinement",
	"scikit-learn": "scikit-learn",
	"clip": "clip",
	}

Affected ones are:
onnxruntime-gpu
opencv-python
Pillow
segmentation-refinement
scikit-learn

No module named 'segmentation_refinement'

Error loading script: main.py
Traceback (most recent call last):
File "D:\Stable_Diffusion\stable-diffusion-webui\modules\scripts.py", line 256, in load_scripts
script_module = script_loading.load_module(scriptfile.path)
File "D:\Stable_Diffusion\stable-diffusion-webui\modules\script_loading.py", line 11, in load_module
module_spec.loader.exec_module(module)
File "", line 883, in exec_module
File "", line 241, in _call_with_frames_removed
File "D:\Stable_Diffusion\stable-diffusion-webui\extensions\PBRemTools\scripts\main.py", line 12, in
from scripts.td_abg import get_foreground
File "D:\Stable_Diffusion\stable-diffusion-webui\extensions\PBRemTools\scripts\td_abg.py", line 15, in
import segmentation_refinement as refine
ModuleNotFoundError: No module named 'segmentation_refinement'

Error loading script: sa_mask.py
Traceback (most recent call last):
File "D:\Stable_Diffusion\stable-diffusion-webui\modules\scripts.py", line 256, in load_scripts
script_module = script_loading.load_module(scriptfile.path)
File "D:\Stable_Diffusion\stable-diffusion-webui\modules\script_loading.py", line 11, in load_module
module_spec.loader.exec_module(module)
File "", line 883, in exec_module
File "", line 241, in _call_with_frames_removed
File "D:\Stable_Diffusion\stable-diffusion-webui\extensions\PBRemTools\scripts\sa_mask.py", line 13, in
from segment_anything import SamAutomaticMaskGenerator, sam_model_registry
ModuleNotFoundError: No module named 'segment_anything'

Error loading script: td_abg.py
Traceback (most recent call last):
File "D:\Stable_Diffusion\stable-diffusion-webui\modules\scripts.py", line 256, in load_scripts
script_module = script_loading.load_module(scriptfile.path)
File "D:\Stable_Diffusion\stable-diffusion-webui\modules\script_loading.py", line 11, in load_module
module_spec.loader.exec_module(module)
File "", line 883, in exec_module
File "", line 241, in _call_with_frames_removed
File "D:\Stable_Diffusion\stable-diffusion-webui\extensions\PBRemTools\scripts\td_abg.py", line 15, in
import segmentation_refinement as refine
ModuleNotFoundError: No module named 'segmentation_refinement'

====================

I've met all the requirements and completed the installation, but there's an error message here...... I don't know what I'm missing.

Explanation of parameters?

Here's what I've grokked from a combination of playing around and reading the source code. Is this roughly accurate?

Segment Anything & CLIP

1. enabled

This is a misnomer. If this is enabled, Segment Anything & CLIP are both enabled.

If this is disabled, it segments the image using anime-seg instead, and doesn't do anything CLIP-like.

2. Model

s == small, l == large, h == huge? I kind of wish they had chosen 'xl' instead, as that's a more common term. If you don't have anything here, you need to download the models and put them in the appropriate folder (see readme).

3. segmentation prompt

Beware that this does not do anything unless both a) Segment Anything & CLIP are enabled, and b) either tile division BG remover or cascadePSP is enabled.

This can be blank, in which case in practice it selects regions of the image that are clearly recognizable (typically foreground). See below.

This can also be a prompt that CLIP can grok. You can flip to img2img and use the 'interrogate CLIP' button on an image to get an idea of the sorts of prompts that CLIP generates, though I've been getting best results with extremely short prompts ("person", for instance)

4. predicted_iou_threshold

5. stability_score_threshold

6. clip_threshold

These all only affect Segment Anything & CLIP.

It's easier to describe these together. Looking at the code, roughly speaking what happens is:

a. Segment Anything is run to segment the image into a bunch of regions.
b. Segment Anything also returns some metadata about said regions, including a couple of different measures of 'quality' of regions. Among these measures are predicted_iou and stability_score.
c. Any regions that have a predicted_iou or stability_score lower than predicted_iou_threshold or stability_score_threshold, respectively, are filtered out and ignored.
d. If there is a segmentation prompt, CLIP is run on all remaining regions, and any region that CLIP thinks has a similarity of less than clip_threshold to the segmentation prompt is filtered out and ignored.
e. The union of all remaining regions are taken as the resulting mask.

So all told:

a. First run with no segmentation prompt.
b. Adjust predicted_iou_threshold and/or stability_score_threshold down until you include the entire subject, and then back up until you are excluding as much of the rest of the image as possible.
c. Re-add the segmentation prompt.
d. Adjust clip_threshold down until you are including the entire subject, then back up until you are excluding as much of the rest of the image as possible.
e. Hopefully you're done. If you get bad results, you need to tweak the segmentation prompt and retry from d.

tile division BG Removers

This is an approach to refine an existing mask, using the mask and the input image.

7. enabled

If enabled, td-abg will be run at the end. Yes, this is out of order. If disabled, it will not be run.

The rest of this is covered fairly well here.

General tuning:

1. The more divisions you have, the more likely is that you'll get noise just from random chance. But too few divisions and you'll get poor results - if part A of the subject shares similar colors as part B of the background, you don't want them in the same cell.
2. The more clusters you have, the more likely is that you'll get noise just from random chance. But too few clusters and you'll get blobs from parts of the subject getting lumped together with parts of the background.
3. The alpha threshold is the only setting that can shrink the mask. Everything else expands it. So you can start by setting mask content ratio to 0 to effectively disable mask expansion, and adjusting alpha threshold until nothing outside the subject is in the mask, and then adjusting the other settings from there.

13. enabled

If enabled, Cascade-PSP will be run after segmentation (and before td-abg, if enabled). If disabled, it will not be run.

14. fast

15. Memory usage

It's easier to cover both of these together. 'Memory usage is a misnomer'. A better description might be 'maximum native input resolution'.

The TL;DR is roughly:

a. If the input image is less than 'memory usage' pixels wide/high, you may as well use fast mode.
b. If the input image is more than 'memory usage' pixels wide/high, you have three options:
b.i. use fast mode, with poor image quality.
b.ii. use fast mode but increase 'memory usage' to a larger value to accommodate the image.
b.iii. disable fast mode, with lower memory usage than ii but likely better results.

If you have the patience, and gpu memory, ~1300 is the optimum value for memory usage.

---

If the above is accurate there's a bunch of UI cleanup I could do.

Hi, any plan to support batch processing?

not working!

hello problem not working! manjaro linux

`Error running install.py for extension /home/tobias/stable-diffusion-webui/extensions/PBRemTools.
Command: "/home/tobias/stable-diffusion-webui/venv/bin/python3" "/home/tobias/stable-diffusion-webui/extensions/PBRemTools/install.py"
Error code: 1
stdout: Installing onnxruntime for PBRemTools

stderr: Traceback (most recent call last):
File "/home/tobias/stable-diffusion-webui/extensions/PBRemTools/install.py", line 16, in
launch.run_pip(f'install {target}', desc=f'{name} for PBRemTools')
File "/home/tobias/stable-diffusion-webui/modules/launch_utils.py", line 124, in run_pip
return run(f'"{python}" -m pip {command} --prefer-binary{index_url_line}', desc=f"Installing {desc}", errdesc=f"Couldn't install {desc}", live=live)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/tobias/stable-diffusion-webui/modules/launch_utils.py", line 101, in run
raise RuntimeError("\n".join(error_bits))
RuntimeError: Couldn't install onnxruntime for PBRemTools.
Command: "/home/tobias/stable-diffusion-webui/venv/bin/python3" -m pip install onnxruntime-gpu==1.14.0 --prefer-binary
Error code: 1
stderr: ERROR: Could not find a version that satisfies the requirement onnxruntime-gpu==1.14.0 (from versions: 1.15.0)
ERROR: No matching distribution found for onnxruntime-gpu==1.14.0
Error loading script: api.py
Traceback (most recent call last):
File "/home/tobias/stable-diffusion-webui/modules/scripts.py", line 263, in load_scripts
script_module = script_loading.load_module(scriptfile.path)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/tobias/stable-diffusion-webui/modules/script_loading.py", line 10, in load_module
module_spec.loader.exec_module(module)
File "", line 940, in exec_module
File "", line 241, in _call_with_frames_removed
File "/home/tobias/stable-diffusion-webui/extensions/PBRemTools/scripts/api.py", line 9, in
from scripts.main import process_image, model_list
File "/home/tobias/stable-diffusion-webui/extensions/PBRemTools/scripts/main.py", line 15, in
from scripts.td_abg import get_foreground
File "/home/tobias/stable-diffusion-webui/extensions/PBRemTools/scripts/td_abg.py", line 5, in
from sklearn.cluster import KMeans, MiniBatchKMeans
ModuleNotFoundError: No module named 'sklearn'

Error loading script: main.py
Traceback (most recent call last):
File "/home/tobias/stable-diffusion-webui/modules/scripts.py", line 263, in load_scripts
script_module = script_loading.load_module(scriptfile.path)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/tobias/stable-diffusion-webui/modules/script_loading.py", line 10, in load_module
module_spec.loader.exec_module(module)
File "", line 940, in exec_module
File "", line 241, in _call_with_frames_removed
File "/home/tobias/stable-diffusion-webui/extensions/PBRemTools/scripts/main.py", line 15, in
from scripts.td_abg import get_foreground
File "/home/tobias/stable-diffusion-webui/extensions/PBRemTools/scripts/td_abg.py", line 5, in
from sklearn.cluster import KMeans, MiniBatchKMeans
ModuleNotFoundError: No module named 'sklearn'

Error loading script: sa_mask.py
Traceback (most recent call last):
File "/home/tobias/stable-diffusion-webui/modules/scripts.py", line 263, in load_scripts
script_module = script_loading.load_module(scriptfile.path)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/tobias/stable-diffusion-webui/modules/script_loading.py", line 10, in load_module
module_spec.loader.exec_module(module)
File "", line 940, in exec_module
File "", line 241, in _call_with_frames_removed
File "/home/tobias/stable-diffusion-webui/extensions/PBRemTools/scripts/sa_mask.py", line 13, in
from segment_anything import SamAutomaticMaskGenerator, sam_model_registry
ModuleNotFoundError: No module named 'segment_anything'

Error loading script: td_abg.py
Traceback (most recent call last):
File "/home/tobias/stable-diffusion-webui/modules/scripts.py", line 263, in load_scripts
script_module = script_loading.load_module(scriptfile.path)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/tobias/stable-diffusion-webui/modules/script_loading.py", line 10, in load_module
module_spec.loader.exec_module(module)
File "", line 940, in exec_module
File "", line 241, in _call_with_frames_removed
File "/home/tobias/stable-diffusion-webui/extensions/PBRemTools/scripts/td_abg.py", line 5, in
from sklearn.cluster import KMeans, MiniBatchKMeans
ModuleNotFoundError: No module named 'sklearn'`

Setting sliders have no effect

Not sure if prompt issue is related: #9

Do sliders only cause effects if the prompts work? If so, then the issue is probably just caused by 9, otherwise there might be a problem with slider parameters as well. Screenshots below.

Ideally was hoping to fine tune accuracy of cutout around feet:

Output same every time regardless:

EDIT: Original img for reference:

Extension Tab UI Duplication

Extension Tab UI Duplication

tab duplication at latest a1111 1.2.1

It seems that api mode by script_callbacks.on_app_started cause this duplication because evict api.py resolve this problem in my env. (but I do not survey deeply.)

I think that this need to improve callback register or implement api mode as a independent script from a1111.

Error if no masks found

If there are no masks found, this errors out here, due to said reduce having no default value.

I can reproduce this fairly easily:

1. Leave settings on defaults aside from the below.
2. Input this image.
3. Enable Segment Anything & CLIP.
4. Set SAM model to sam_vit_h_4b8939.pth.
5. Set predicted_iou_threshold / stability_score_threshold / clip_threshold to 1.
6. Set a segmentation prompt of 'asdf'.
7. Enable CascadePSP
8. Set CascadePSP to fast.
9. Generate.

(Most of these steps are probably unnecessary. Ditto, I realize that setting a threshold to 1 is somewhat silly.)

Said error is also fairly ungraceful in the UI: a red 'Error' and that's it.

Maybe either a) pass a default all-ones mask or b) if no masks pass the threshold then pick one mask with the highest confidence?

[Feature Request] Custom Background Color

Everything is simple. You choose a color and he stupidly makes the background of that color.

Request for Batch Background Removal and API Control Functionality in Plugin

Hello,

I would like to request a feature for the current background removal plugin. Currently, it only allows manual removal of images one by one. I suggest adding a batch removal feature and an API for controlling the removal process. This would be very helpful for users who need to remove backgrounds from multiple images at once.

Thank you.

Feature suggestion: Interactive Selection

Pretty much as the title suggests. Being able to click/highlight the parts of the Image you want to segment out, like in Facebooks own demo.

Enforce onnxruntime 1.14.0

Currently, onnxruntime 1.14.1 is breaking other extensions like rembg. We've had extensive discussions on this over at Automatic.

Could you enforce version 1.14.0? It doesn't seem to affect PBRemTools in any way and maintains compatibility across the environment.

PNG to SVG converter

Is there a way we can turn pixels to svg locally?

I hope to add batch processing as soon as possible, it will help us a lot

ERROR message - Can not use the extension

Added the extension and my first attempt to use it. I keep getting the below message. Is there something I'm doing wrong?

Traceback (most recent call last):
File "C:\Users\XXXX\Documents\WebUI-Stable-Diffusion\stable-diffusion-webui\venv\lib\site-packages\gradio\routes.py", line 394, in run_predict
output = await app.get_blocks().process_api(
File "C:\Users\XXXX\Documents\WebUI-Stable-Diffusion\stable-diffusion-webui\venv\lib\site-packages\gradio\blocks.py", line 1075, in process_api
result = await self.call_function(
File "C:\Users\XXXX\Documents\WebUI-Stable-Diffusion\stable-diffusion-webui\venv\lib\site-packages\gradio\blocks.py", line 884, in call_function
prediction = await anyio.to_thread.run_sync(
File "C:\Users\XXXX\Documents\WebUI-Stable-Diffusion\stable-diffusion-webui\venv\lib\site-packages\anyio\to_thread.py", line 31, in run_sync
return await get_asynclib().run_sync_in_worker_thread(
File "C:\Users\XXXX\Documents\WebUI-Stable-Diffusion\stable-diffusion-webui\venv\lib\site-packages\anyio_backends_asyncio.py", line 937, in run_sync_in_worker_thread
return await future
File "C:\Users\XXXX\Documents\WebUI-Stable-Diffusion\stable-diffusion-webui\venv\lib\site-packages\anyio_backends_asyncio.py", line 867, in run
result = context.run(func, *args)
TypeError: processing() takes 10 positional arguments but 12 were given

ValueError: zero-size array to reduction operation minimum which has no identity

When I enter segmentation prompt ,For example“girl” It will report an error.，

To create a public link, set share=True in launch().
[AgentScheduler] Task queue is empty
[AgentScheduler] Registering APIs
Startup time: 27.9s (import torch: 1.4s, import gradio: 1.3s, import ldm: 0.7s, other imports: 0.7s, load scripts: 6.8s, create ui: 13.7s, gradio launch: 2.5s, add APIs: 0.1s, app_started_callback: 0.5s).
Traceback (most recent call last):
File "/root/stable-diffusion-webui/venv/lib/python3.10/site-packages/gradio/routes.py", line 422, in run_predict
output = await app.get_blocks().process_api(
File "/root/stable-diffusion-webui/venv/lib/python3.10/site-packages/gradio/blocks.py", line 1323, in process_api
result = await self.call_function(
File "/root/stable-diffusion-webui/venv/lib/python3.10/site-packages/gradio/blocks.py", line 1051, in call_function
prediction = await anyio.to_thread.run_sync(
File "/root/stable-diffusion-webui/venv/lib/python3.10/site-packages/anyio/to_thread.py", line 33, in run_sync
return await get_asynclib().run_sync_in_worker_thread(
File "/root/stable-diffusion-webui/venv/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 877, in run_sync_in_worker_thread
return await future
File "/root/stable-diffusion-webui/venv/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 807, in run
result = context.run(func, *args)
File "/root/stable-diffusion-webui/extensions/PBRemTools/scripts/tools.py", line 32, in processing
processed = process_image(single_image, *rem_args)
File "/root/stable-diffusion-webui/extensions/PBRemTools/scripts/tools.py", line 80, in process_image
mask, image = get_foreground(image, *rem_args)
File "/root/stable-diffusion-webui/extensions/PBRemTools/scripts/td_abg.py", line 108, in get_foreground
mask = get_sa_mask(img, query, model_name, predicted_iou_threshold, stability_score_threshold, clip_threshold)
File "/root/stable-diffusion-webui/extensions/PBRemTools/scripts/sa_mask.py", line 168, in get_sa_mask
combined_mask = np.minimum.reduce(mask_list)
ValueError: zero-size array to reduction operation minimum which has no identity