Stable Diffusion XL优化终极指南

python -m venv .venv

# Unixsource .venv/bin/activate# Windows.venv\Scripts\activate

pip install torch torchvision --index-url https://download.pytorch.org/whl/cu118pip install transformers accelerate diffusers

queue = []# Photorealistic portrait (Portrait)queue.extend([{ 'prompt': '3/4 shot, candid photograph of a beautiful 30 year old redhead woman with messy dark hair, peacefully sleeping in her bed, night, dark, light from window, dark shadows, masterpiece, uhd, moody', 'seed': 877866765,}])# Creative interior image (Interior)queue.extend([{ 'prompt': 'futuristic living room with big windows, brown sofas, coffee table, plants, cyberpunk city, concept art, earthy colors', 'seed': 5567822456,}])# Macro photography (Macro)queue.extend([{ 'prompt': 'macro shot of a bee collecting nectar from lavender flowers', 'seed': 2257899453,}])# Rendered 3D image (3D)queue.extend([{ 'prompt': '3d rendered isometric fiji island beach, 3d tile, polygon, cartoony, mobile game', 'seed': 987867834,}])

from time import perf_counter# Import libraries# import ...
# Define prompts# queue = []# queue.extend ...
for i, generation in enumerate(queue, start=1): # We start the counter image_start = perf_counter()
 # Generate and save image # ...
 # We stop the counter and save the result generation['total_time'] = perf_counter() - image_start
# Print the generation time of each imageimages_totals = ', '.join(map(lambda generation: str(round(generation['total_time'], 1)), queue))print('Image time:', images_totals)
# Print the average timeimages_average = round(sum(generation['total_time'] for generation in queue) / len(queue), 1)print('Average image time:', images_average)

max_memory = round(torch.cuda.max_memory_allocated(device='cuda') / 1000000000, 2)print('Max. memory used:', max_memory, 'GB')

# Load the model on the graphics cardpipe = AutoPipelineForText2Image.from_pretrained( 'stabilityai/stable-diffusion-xl-base-1.0', use_safetensors=True, torch_dtype=torch.float16, variant='fp16',).to('cuda')
# Create a generatorgenerator = torch.Generator(device='cuda')
# Start a loop to process prompts one by onefor i, generation in enumerate(queue, start=1): # Assign the seed to the generator generator.manual_seed(generation['seed'])
 # Create the image image = pipe( prompt=generation['prompt'], generator=generator, ).images[0]
 # Save the image image.save(f'image_{i}.png')

pipe = AutoPipelineForText2Image.from_pretrained( 'stabilityai/stable-diffusion-xl-base-1.0', use_safetensors=True, torch_dtype=torch.float16, variant='fp16',).to('cuda')
generator = torch.Generator(device='cuda')
for i, generation in enumerate(queue, start=1): generator.manual_seed(generation['seed'])
 image = pipe( prompt=generation['prompt'], generator=generator, ).images[0]
 image.save(f'image_{i}.png')

torch.backends.cudnn.allow_tf32 = False# it's already disabled by default# torch.backends.cuda.matmul.allow_tf32 = False
pipe = AutoPipelineForText2Image.from_pretrained( 'stabilityai/stable-diffusion-xl-base-1.0', use_safetensors=True, torch_dtype=torch.float16, variant='fp16',).to('cuda')
generator = torch.Generator(device='cuda')
for i, generation in enumerate(queue, start=1): generator.manual_seed(generation['seed'])
 image = pipe( prompt=generation['prompt'], generator=generator, ).images[0]
 image.save(f'image_{i}.png')

# it's already activated by default# torch.backends.cudnn.allow_tf32 = Truetorch.backends.cuda.matmul.allow_tf32 = True
pipe = AutoPipelineForText2Image.from_pretrained( 'stabilityai/stable-diffusion-xl-base-1.0', use_safetensors=True,).to('cuda')
generator = torch.Generator(device='cuda')
for i, generation in enumerate(queue, start=1): generator.manual_seed(generation['seed'])
 image = pipe( prompt=generation['prompt'], generator=generator, ).images[0]
 image.save(f'image_{i}.png')

model_cpu_offload_seq = "text_encoder->text_encoder_2->image_encoder->unet->vae"

pipe = AutoPipelineForText2Image.from_pretrained( 'stabilityai/stable-diffusion-xl-base-1.0', use_safetensors=True, torch_dtype=torch.float16, variant='fp16',).to('cuda')pipe.enable_model_cpu_offload()generator = torch.Generator(device='cuda')for i, generation in enumerate(queue, start=1): generator.manual_seed(generation['seed']) image = pipe( prompt=generation['prompt'], generator=generator, ).images[0] image.save(f'image_{i}.png')

pipe = AutoPipelineForText2Image.from_pretrained( # ...).to('cuda')

pipe = AutoPipelineForText2Image.from_pretrained( 'stabilityai/stable-diffusion-xl-base-1.0', use_safetensors=True, torch_dtype=torch.float16, variant='fp16',)
pipe.enable_sequential_cpu_offload()
generator = torch.Generator(device='cuda')
for i, generation in enumerate(queue, start=1): generator.manual_seed(generation['seed'])
 image = pipe( prompt=generation['prompt'], generator=generator, ).images[0]
 image.save(f'image_{i}.png')

pipe = AutoPipelineForText2Image.from_pretrained( 'stabilityai/stable-diffusion-xl-base-1.0', use_safetensors=True, torch_dtype=torch.float16, variant='fp16', unet=None, vae=None,).to('cuda')

def encode_prompt(prompts, tokenizers, text_encoders): embeddings_list = [] for prompt, tokenizer, text_encoder in zip(prompts, tokenizers, text_encoders): cond_input = tokenizer( prompt, max_length=tokenizer.model_max_length, padding='max_length', truncation=True, return_tensors='pt', ) prompt_embeds = text_encoder(cond_input.input_ids.to('cuda'), output_hidden_states=True) pooled_prompt_embeds = prompt_embeds[0] embeddings_list.append(prompt_embeds.hidden_states[-2]) prompt_embeds = torch.concat(embeddings_list, dim=-1) negative_prompt_embeds = torch.zeros_like(prompt_embeds) negative_pooled_prompt_embeds = torch.zeros_like(pooled_prompt_embeds) bs_embed, seq_len, _ = prompt_embeds.shape prompt_embeds = prompt_embeds.repeat(1, 1, 1) prompt_embeds = prompt_embeds.view(bs_embed * 1, seq_len, -1) seq_len = negative_prompt_embeds.shape[1] negative_prompt_embeds = negative_prompt_embeds.repeat(1, 1, 1) negative_prompt_embeds = negative_prompt_embeds.view(1 * 1, seq_len, -1) pooled_prompt_embeds = pooled_prompt_embeds.repeat(1, 1).view(bs_embed * 1, -1) negative_pooled_prompt_embeds = negative_pooled_prompt_embeds.repeat(1, 1).view(bs_embed * 1, -1) return prompt_embeds, negative_prompt_embeds, pooled_prompt_embeds, negative_pooled_prompt_embeds

import gcfrom transformers import CLIPTokenizer, CLIPTextModel, CLIPTextModelWithProjection# ...tokenizer = CLIPTokenizer.from_pretrained( 'stabilityai/stable-diffusion-xl-base-1.0', subfolder='tokenizer',)text_encoder = CLIPTextModel.from_pretrained( 'stabilityai/stable-diffusion-xl-base-1.0', subfolder='text_encoder', use_safetensors=True, torch_dtype=torch.float16, variant='fp16',).to('cuda')tokenizer_2 = CLIPTokenizer.from_pretrained( 'stabilityai/stable-diffusion-xl-base-1.0', subfolder='tokenizer_2',)text_encoder_2 = CLIPTextModelWithProjection.from_pretrained( 'stabilityai/stable-diffusion-xl-base-1.0', subfolder='text_encoder_2', use_safetensors=True, torch_dtype=torch.float16, variant='fp16',).to('cuda')

with torch.no_grad(): for generation in queue: generation['embeddings'] = encode_prompt( [generation['prompt'], generation['prompt']], [tokenizer, tokenizer_2], [text_encoder, text_encoder_2], )

del tokenizer, text_encoder, tokenizer_2, text_encoder_2gc.collect()torch.cuda.empty_cache()

pipe = AutoPipelineForText2Image.from_pretrained( 'stabilityai/stable-diffusion-xl-base-1.0', use_safetensors=True, torch_dtype=torch.float16, variant='fp16', tokenizer=None, text_encoder=None, tokenizer_2=None, text_encoder_2=None,).to('cuda')

for generation in queue: pipe( prompt_embeds=generation['embeddings'][0], negative_prompt_embeds =generation['embeddings'][1], pooled_prompt_embeds=generation['embeddings'][2], negative_pooled_prompt_embeds=generation['embeddings'][3], output_type='latent', )

generator = torch.Generator(device='cuda')
for i, generation in enumerate(queue, start=1): generator.manual_seed(generation['seed'])
 generation['latents'] = pipe( prompt_embeds=generation['embeddings'][0], negative_prompt_embeds =generation['embeddings'][1], pooled_prompt_embeds=generation['embeddings'][2], negative_pooled_prompt_embeds=generation['embeddings'][3], generator=generator, output_type='latent', ).images # We do not access images[0], but the entire tensor

del pipe.unetgc.collect()torch.cuda.empty_cache()

pipe.upcast_vae()
with torch.no_grad(): for i, generation in enumerate(queue, start=1): generation['latents'] = generation['latents'].to(next(iter(pipe.vae.post_quant_conv.parameters())).dtype)
 image = pipe.vae.decode( generation['latents'] / pipe.vae.config.scaling_factor, return_dict=False, )[0]
 image = pipe.image_processor.postprocess(image, output_type='pil')[0] image.save(f'image_{i}.png')

pip install stable-fastpip install torch torchvision triton xformers --index-url https://download.pytorch.org/whl/cu118

import xformersimport tritonfrom sfast.compilers.diffusion_pipeline_compiler import (compile, CompilationConfig)
# ...
pipe = AutoPipelineForText2Image.from_pretrained( 'stabilityai/stable-diffusion-xl-base-1.0', use_safetensors=True, torch_dtype=torch.float16, variant='fp16',).to('cuda')
config = CompilationConfig.Default()
config.enable_xformers = Trueconfig.enable_triton = Trueconfig.enable_cuda_graph = True
pipe = compile(pipe, config)
generator = torch.Generator(device='cuda')
for i, generation in enumerate(queue, start=1): generator.manual_seed(generation['seed'])
 image = pipe( prompt=generation['prompt'], generator=generator, ).images[0]
 image.save(f'image_{i}.png')

pip install deepcache

from DeepCache import DeepCacheSDHelper
# ...
pipe = AutoPipelineForText2Image.from_pretrained( 'stabilityai/stable-diffusion-xl-base-1.0', use_safetensors=True, torch_dtype=torch.float16, variant='fp16',).to('cuda')
helper = DeepCacheSDHelper(pipe=pipe)helper.set_params(cache_interval=3, cache_branch_id=0)helper.enable()
generator = torch.Generator(device='cuda')
for i, generation in enumerate(queue, start=1): generator.manual_seed(generation['seed'])
 image = pipe( prompt=generation['prompt'], generator=generator, ).images[0]
 image.save(f'image_{i}.png')

# 克隆整个仓库或从此文件夹下载文件# https://github.com/rajeevsrao/TensorRT/tree/release/8.6/demo/Diffusion# 像往常一样创建并激活一个虚拟环境python -m venv .venv## Unixsource .venv/bin/activate## Windows.venv\Scripts\activate# 安装所需的库pip install torch torchvision --index-url https://download.pytorch.org/whl/cu118pip install transformers accelerate diffusers cuda-python nvtx onnx colored scipy polygraphypip install --pre --extra-index-url https://pypi.nvidia.com tensorrtpip install --pre --extra-index-url https://pypi.ngc.nvidia.com onnx_graphsurgeon# 可以使用以下行验证 TensorRT 是否正确安装python -c "import tensorrt; print(tensorrt.__version__)"# 9.3.0.post12.dev1# 进行推理python demo_txt2img_xl.py "macro shot of a bee collecting nectar from lavender flowers"

RuntimeError: Windows not yet supported for torch.compile

pipe = AutoPipelineForText2Image.from_pretrained( 'stabilityai/stable-diffusion-xl-base-1.0', use_safetensors=True, torch_dtype=torch.float16, variant='fp16',).to('cuda')
pipe.unet = torch.compile(pipe.unet, mode='reduce-overhead', fullgraph=True)
generator = torch.Generator(device='cuda')
for i, generation in enumerate(queue, start=1): generator.manual_seed(generation['seed'])
 image = pipe( prompt=generation['prompt'], generator=generator, ).images[0]
 image.save(f'image_{i}.png')

pip install --pre oneflow -f https://github.com/siliconflow/oneflow_releases/releases/expanded_assets/community_cu118
pip install --pre onediff

RuntimeError: This package is a placeholder. Please install oneflow following the instructions in https://github.com/Oneflow-Inc/oneflow#install-oneflow

import oneflow as flowfrom onediff.infer_compiler import oneflow_compile
# ...
pipe = AutoPipelineForText2Image.from_pretrained( 'stabilityai/stable-diffusion-xl-base-1.0', use_safetensors=True, torch_dtype=torch.float16, variant='fp16',).to('cuda')
pipe.unet = oneflow_compile(pipe.unet)
generator = torch.Generator(device='cuda')
with flow.autocast('cuda'): for i, generation in enumerate(queue, start=1): generator.manual_seed(generation['seed'])
 image = pipe( prompt=generation['prompt'], generator=generator, ).images[0]
 image.save(f'image_{i}.png')

pipe = AutoPipelineForText2Image.from_pretrained( 'stabilityai/stable-diffusion-xl-base-1.0', use_safetensors=True, torch_dtype=torch.float16, variant='fp16',).to('cuda')
pipe.unet.to(memory_format=torch.channels_last)
generator = torch.Generator(device='cuda')
for i, generation in enumerate(queue, start=1): generator.manual_seed(generation['seed'])
 image = pipe( prompt=generation['prompt'], generator=generator, ).images[0]
 image.save(f'image_{i}.png')

print(pipe.unet.conv_out.state_dict()['weight'].stride())

pipe = AutoPipelineForText2Image.from_pretrained( 'stabilityai/stable-diffusion-xl-base-1.0', use_safetensors=True, torch_dtype=torch.float16, variant='fp16',).to('cuda')
pipe.enable_freeu(s1=0.9, s2=0.2, b1=1.3, b2=1.4)
generator = torch.Generator(device='cuda')
for i, generation in enumerate(queue, start=1): generator.manual_seed(generation['seed'])
 image = pipe( prompt=generation['prompt'], generator=generator, ).images[0]
 image.save(f'image_{i}.png')

from diffusers import AutoPipelineForText2Image, AutoencoderKL
# ...
vae = AutoencoderKL.from_pretrained( 'madebyollin/sdxl-vae-fp16-fix', use_safetensors=True, torch_dtype=torch.float16,).to('cuda')
pipe = AutoPipelineForText2Image.from_pretrained( 'stabilityai/stable-diffusion-xl-base-1.0', use_safetensors=True, torch_dtype=torch.float16, variant='fp16', vae=vae,).to('cuda')
generator = torch.Generator(device='cuda')
for i, generation in enumerate(queue, start=1): generator.manual_seed(generation['seed'])
 image = pipe( prompt=generation['prompt'], generator=generator, ).images[0]
 image.save(f'image_{i}.png')

pipe = AutoPipelineForText2Image.from_pretrained( 'stabilityai/stable-diffusion-xl-base-1.0', use_safetensors=True, torch_dtype=torch.float16, variant='fp16',).to('cuda')
pipe.enable_vae_slicing()
generator = torch.Generator(device='cuda')
for i, generation in enumerate(queue, start=1): generator.manual_seed(generation['seed'])
 image = pipe( prompt=generation['prompt'], generator=generator, ).images[0]
 image.save(f'image_{i}.png')

pipe = AutoPipelineForText2Image.from_pretrained( 'stabilityai/stable-diffusion-xl-base-1.0', use_safetensors=True, torch_dtype=torch.float16, variant='fp16',).to('cuda')
pipe.enable_vae_tiling()
generator = torch.Generator(device='cuda')
for i, generation in enumerate(queue, start=1): generator.manual_seed(generation['seed'])
 image = pipe( prompt=generation['prompt'], generator=generator, height=4096, width=4096, ).images[0]
 image.save(f'image_{i}.png')

from diffusers import AutoPipelineForText2Image, AutoencoderTiny
# ...
vae = AutoencoderTiny.from_pretrained( 'madebyollin/taesdxl', use_safetensors=True, torch_dtype=torch.float16,).to('cuda')
pipe = AutoPipelineForText2Image.from_pretrained( 'stabilityai/stable-diffusion-xl-base-1.0', use_safetensors=True, torch_dtype=torch.float16, variant='fp16', vae=vae,).to('cuda')
generator = torch.Generator(device='cuda')
for i, generation in enumerate(queue, start=1): generator.manual_seed(generation['seed'])
 image = pipe( prompt=generation['prompt'], generator=generator, ).images[0]
 image.save(f'image_{i}.png')

pipe = AutoPipelineForText2Image.from_pretrained( 'stabilityai/stable-diffusion-xl-base-1.0', use_safetensors=True, torch_dtype=torch.float16, variant='fp16',).to('cuda')
generator = torch.Generator(device='cuda')
for i, generation in enumerate(queue, start=1): generator.manual_seed(generation['seed'])
 image = pipe( prompt=generation['prompt'], num_inference_steps=30, generator=generator, ).images[0]
 image.save(f'image_{i}.png')

# As we're using classifier-free guidance, we duplicate the tensor to avoid making two passes# One pass will be for the conditioned values and another for the unconditioned valueslatent_model_input = torch.cat([latents] * 2)

pipe = AutoPipelineForText2Image.from_pretrained( 'stabilityai/stable-diffusion-xl-base-1.0', use_safetensors=True, torch_dtype=torch.float16, variant='fp16',).to('cuda')
def callback_dynamic_cfg(pipe, step_index, timestep, callback_kwargs): if step_index == int(pipe.num_timesteps * 0.5): callback_kwargs['prompt_embeds'] = callback_kwargs['prompt_embeds'].chunk(2)[-1] callback_kwargs['add_text_embeds'] = callback_kwargs['add_text_embeds'].chunk(2)[-1] callback_kwargs['add_time_ids'] = callback_kwargs['add_time_ids'].chunk(2)[-1] pipe._guidance_scale = 0.0
 return callback_kwargs
generator = torch.Generator(device='cuda')
for i, generation in enumerate(queue, start=1): generator.manual_seed(generation['seed'])
 image = pipe( prompt=generation['prompt'], generator=generator, callback_on_step_end=callback_dynamic_cfg, callback_on_step_end_tensor_inputs=['prompt_embeds', 'add_text_embeds', 'add_time_ids'], ).images[0]
 image.save(f'image_{i}.png')

for generation in queue: image = base(generation['prompt'], output_type='latent').images refiner(generation['prompt'], image=image)

from diffusers import AutoPipelineForText2Image, AutoPipelineForImage2Image
# ...
base = AutoPipelineForText2Image.from_pretrained( 'stabilityai/stable-diffusion-xl-base-1.0', use_safetensors=True, torch_dtype=torch.float16, variant='fp16',).to('cuda')
refiner = AutoPipelineForImage2Image.from_pretrained( 'stabilityai/stable-diffusion-xl-refiner-1.0', use_safetensors=True, torch_dtype=torch.float16, variant='fp16',).to('cuda')
generator = torch.Generator(device='cuda')
for i, generation in enumerate(queue, start=1): generator.manual_seed(generation['seed'])
 image = base( prompt=generation['prompt'], generator=generator, num_inference_steps=50, denoising_end=0.8, output_type='latent', ).images # Remember that here we do not access images[0], but the entire tensor
 image = refiner( prompt=generation['prompt'], generator=generator, num_inference_steps=50, denoising_start=0.8, image=image, ).images[0]
 image.save(f'image_{i}.png')

from diffusers import AutoPipelineForText2Image, AutoPipelineForImage2Image
# ...
base = AutoPipelineForText2Image.from_pretrained( 'stabilityai/stable-diffusion-xl-base-1.0', use_safetensors=True, torch_dtype=torch.float16, variant='fp16',).to('cuda')
refiner = AutoPipelineForImage2Image.from_pretrained( 'stabilityai/stable-diffusion-xl-refiner-1.0', use_safetensors=True, torch_dtype=torch.float16, variant='fp16',).to('cuda')
generator = torch.Generator(device='cuda')
for i, generation in enumerate(queue, start=1): generator.manual_seed(generation['seed'])
 image = base( prompt=generation['prompt'], generator=generator, num_inference_steps=50, ).images[0]
 image = refiner( prompt=generation['prompt'], generator=generator, num_inference_steps=10, image=image, ).images[0]
 image.save(f'image_{i}.png')