CODEWITHSUNDEEP
torchdiffuserscontrolnet
daisy
daisy

Reputation: 23571

IP-adapter plus face model not working as expected

I came from these two links,

They all mentioned I can preserve face id with the controlnet model.

So I created a simple script to test that,

#!/usr/bin/env python3.10

import os
import cv2
import torch

from diffusers import StableDiffusionXLPipeline, DDIMScheduler, AutoencoderKL
from PIL import Image
from ip_adapter.ip_adapter_faceid import IPAdapterFaceIDXL
from insightface.app import FaceAnalysis
from huggingface_hub import hf_hub_download

device = torch.device('mps') if torch.backends.mps.is_available() else torch.device('cuda')
torchDtype = torch.float32 if torch.backends.mps.is_available() else torch.float16

app = FaceAnalysis(name="buffalo_l", providers=['CUDAExecutionProvider', 'CPUExecutionProvider'])
app.prepare(ctx_id=0, det_size=(640, 640))

image = cv2.imread("./inputs/1.jpg")
faces = app.get(image)
faceid_embeds = torch.from_numpy(faces[0].normed_embedding).unsqueeze(0)

noise_scheduler = DDIMScheduler(
    num_train_timesteps=1000,
    beta_start=0.00085,
    beta_end=0.012,
    beta_schedule="scaled_linear",
    clip_sample=False,
    set_alpha_to_one=False,
    steps_offset=1,
)

pipe = StableDiffusionXLPipeline.from_pretrained(
    "SG161222/RealVisXL_V4.0",
    scheduler=noise_scheduler,
    add_watermarker=False,
    torch_dtype=torchDtype
)
pipe.to(device)
pipe.enable_attention_slicing()

ip_ckpt = hf_hub_download(repo_id="h94/IP-Adapter-FaceID", filename="ip-adapter-faceid_sdxl.bin")
ip_model = IPAdapterFaceIDXL(pipe, ip_ckpt, device)

prompt = "a woman"
negative_prompt = "Watermark, Text, censored, deformed, bad anatomy, disfigured, poorly drawn face, mutated, extra limb, ugly, poorly drawn hands, missing limb, floating limbs, disconnected limbs, disconnected head, malformed hands, long neck, mutated hands and fingers, bad hands, missing fingers, cropped, worst quality, low quality, mutation, poorly drawn, huge calf, fused hand, missing hand, disappearing arms, disappearing thigh, disappearing calf, disappearing legs, fused fingers, abnormal eye proportion, Abnormal hands, abnormal legs, abnormal feet, abnormal fingers"

images = ip_model.generate(
    faceid_embeds,
    prompt,
    negative_prompt, 
    7,
    4,
    None,
    40,
    width=1024,
    height=1024
)

os.makedirs('outputs', 0o755, exists_ok=True)
for i, img in enumerate(images):
    img.save(f"outputs/ipa-{i + 1}.jpg")

But the generated result is nothing like the inputs. What's wrong?

Upvotes: 0

Views: 48

Answers (0)

Related Questions