Comments (9)
Can you provide the full code of local_inference.py
?
from crm.
import argparse
import numpy as np
import gradio as gr
from omegaconf import OmegaConf
import torch
from PIL import Image
import PIL
from pipelines import TwoStagePipeline
from huggingface_hub import hf_hub_download
import os
import rembg
from typing import Any
import json
import os
import json
import argparse
from model import CRM
from inference import generate3d
pipeline = None
rembg_session = rembg.new_session()
def expand_to_square(image, bg_color=(0, 0, 0, 0)):
# expand image to 1:1
width, height = image.size
if width == height:
return image
new_size = (max(width, height), max(width, height))
new_image = Image.new("RGBA", new_size, bg_color)
paste_position = ((new_size[0] - width) // 2, (new_size[1] - height) // 2)
new_image.paste(image, paste_position)
return new_image
def check_input_image(input_image):
if input_image is None:
raise gr.Error("No image uploaded!")
def remove_background(
image: PIL.Image.Image,
rembg_session: Any = None,
force: bool = False,
**rembg_kwargs,
) -> PIL.Image.Image:
do_remove = True
if image.mode == "RGBA" and image.getextrema()[3][0] < 255:
# explain why current do not rm bg
print("alhpa channl not enpty, skip remove background, using alpha channel as mask")
background = Image.new("RGBA", image.size, (0, 0, 0, 0))
image = Image.alpha_composite(background, image)
do_remove = False
do_remove = do_remove or force
if do_remove:
image = rembg.remove(image, session=rembg_session, **rembg_kwargs)
return image
def do_resize_content(original_image: Image, scale_rate):
# resize image content wile retain the original image size
if scale_rate != 1:
# Calculate the new size after rescaling
new_size = tuple(int(dim * scale_rate) for dim in original_image.size)
# Resize the image while maintaining the aspect ratio
resized_image = original_image.resize(new_size)
# Create a new image with the original size and black background
padded_image = Image.new("RGBA", original_image.size, (0, 0, 0, 0))
paste_position = ((original_image.width - resized_image.width) // 2, (original_image.height - resized_image.height) // 2)
padded_image.paste(resized_image, paste_position)
return padded_image
else:
return original_image
def add_background(image, bg_color=(255, 255, 255)):
# given an RGBA image, alpha channel is used as mask to add background color
background = Image.new("RGBA", image.size, bg_color)
return Image.alpha_composite(background, image)
def preprocess_image(image, background_choice, foreground_ratio, backgroud_color):
"""
input image is a pil image in RGBA, return RGB image
"""
print(background_choice)
if background_choice == "Alpha as mask":
background = Image.new("RGBA", image.size, (0, 0, 0, 0))
image = Image.alpha_composite(background, image)
else:
image = remove_background(image, rembg_session, force_remove=True)
image = do_resize_content(image, foreground_ratio)
image = expand_to_square(image)
image = add_background(image, backgroud_color)
return image.convert("RGB")
def gen_image(input_image, seed, scale, step):
global pipeline, model, args
pipeline.set_seed(seed)
rt_dict = pipeline(input_image, scale=scale, step=step)
stage1_images = rt_dict["stage1_images"]
stage2_images = rt_dict["stage2_images"]
np_imgs = np.concatenate(stage1_images, 1)
np_xyzs = np.concatenate(stage2_images, 1)
glb_path, obj_path = generate3d(model, np_imgs, np_xyzs, args.device)
return Image.fromarray(np_imgs), Image.fromarray(np_xyzs), glb_path, obj_path
parser = argparse.ArgumentParser()
parser.add_argument(
"--stage1_config",
type=str,
default="configs/nf7_v3_SNR_rd_size_stroke.yaml",
help="config for stage1",
)
parser.add_argument(
"--stage2_config",
type=str,
default="configs/stage2-v2-snr.yaml",
help="config for stage2",
)
parser.add_argument("--device", type=str, default="cuda")
args = parser.parse_args()
#crm_path = hf_hub_download(repo_id="Zhengyi/CRM", filename="CRM.pth")
crm_path = '.../CRM.pth'
specs = json.load(open("configs/specs_objaverse_total.json"))
model = CRM(specs).to(args.device)
model.load_state_dict(torch.load(crm_path, map_location = args.device), strict=False)
stage1_config = OmegaConf.load(args.stage1_config).config
stage2_config = OmegaConf.load(args.stage2_config).config
stage2_sampler_config = stage2_config.sampler
stage1_sampler_config = stage1_config.sampler
stage1_model_config = stage1_config.models
stage2_model_config = stage2_config.models
#xyz_path = hf_hub_download(repo_id="Zhengyi/CRM", filename="ccm-diffusion.pth")
#pixel_path = hf_hub_download(repo_id="Zhengyi/CRM", filename="pixel-diffusion.pth")
xyz_path = '.../ccm-diffusion.pth'
pixel_path = '.../pixel-diffusion.pth'
stage1_model_config.resume = xyz_path
stage2_model_config.resume = pixel_path
pipeline = TwoStagePipeline(
stage1_model_config,
stage2_model_config,
stage1_sampler_config,
stage2_sampler_config,
device=args.device,
dtype=torch.float16
)
image_path = '.../demo_img.png'
image_input = PIL.Image.open(image_path)
preprocessed_image = preprocess_image(image_input, "Alpha as mask", 1.0, "#7F7F7F")
novel_views, ccms, glb_path, obj_path = gen_image(preprocessed_image, 1234, 0.55, 30)
I didn't modify the codes too much actually.
Thanks
from crm.
The code
stage1_model_config.resume = xyz_path
stage2_model_config.resume = pixel_path
should be changed into the following code?
stage1_model_config.resume = pixel_path
stage2_model_config.resume = xyz_path
from crm.
Thanks for your reply! Interesting, so the checkpoints do need to be swapped. Although the models can be loaded in this way, but I cannot get same output quality as the ones from HF gradio. Do you have any suggestions?
from crm.
Can I see your your 3D result? Also, pixel diffusion is for stage1 and xyz diffusion is for stage2. There is no swap.
from crm.
I have tried two separate runs and got very different novel view generation results:
from crm.
The last colume is your input image? I think the problem results from the unclean background? We assume that the input image is correctly pre-processed into a grey background, otherwise the results will be unpredictable.
from crm.
Oh not exactly, here is the preprocessed image:
So you are suggesting the preprocessed image may be wrongly passed to the generation pipeline? Thanks!
from crm.
Yes, the preprocessed image may be wrongly passed to the generation pipeline. The last image should be exactly the same as the preprocessed image.
from crm.
Related Issues (20)
- Upscaled RBG and CCM ,Tile-Based generation HOT 8
- How do you render an image from flexicube geometry? HOT 1
- Why can't I successfully download the weights file?
- 请您把项目给清晰些,不要霍霍别人 HOT 2
- 关于demo的lego-style HOT 1
- environment problem HOT 1
- 显存最低要求是多少?8GB报错内存不足 HOT 1
- Could you share the filter list of Objaverse?
- problems about the random seed in the code HOT 2
- CUDA out of memory
- gradio app.py error HOT 2
- Multi GPUs
- Seeking Clarification on VAE and CRM Performance in CCM Diffusion HOT 1
- Hi, image as guide
- Is there any plan to release training script? HOT 4
- RuntimeError: Ninja is required to load C++ extensions HOT 4
- version error HOT 2
- Error with Xformers newest version HOT 2
- training code HOT 2
Recommend Projects
-
React
A declarative, efficient, and flexible JavaScript library for building user interfaces.
-
Vue.js
🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.
-
Typescript
TypeScript is a superset of JavaScript that compiles to clean JavaScript output.
-
TensorFlow
An Open Source Machine Learning Framework for Everyone
-
Django
The Web framework for perfectionists with deadlines.
-
Laravel
A PHP framework for web artisans
-
D3
Bring data to life with SVG, Canvas and HTML. 📊📈🎉
-
Recommend Topics
-
javascript
JavaScript (JS) is a lightweight interpreted programming language with first-class functions.
-
web
Some thing interesting about web. New door for the world.
-
server
A server is a program made to process requests and deliver data to clients.
-
Machine learning
Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.
-
Visualization
Some thing interesting about visualization, use data art
-
Game
Some thing interesting about game, make everyone happy.
Recommend Org
-
Facebook
We are working to build community through open source technology. NB: members must have two-factor auth.
-
Microsoft
Open source projects and samples from Microsoft.
-
Google
Google ❤️ Open Source for everyone.
-
Alibaba
Alibaba Open Source for everyone
-
D3
Data-Driven Documents codes.
-
Tencent
China tencent open source team.
from crm.