Very, Very alpha version of a SD3.5M finetune. Posting just to keep a record of training settings/give others a base to start from/proof of concept. 1st version is diffusers. New workflow (You'll want to grab the workflow from one of my images below) to solve corruption issue -- basically, if you put too many tokens into CLIP SD3.5M starts corrupting your image, so breaking the prompt up and feeding directly into L, G, and T5 can fix this. Use tags or barebones prompt for CLIP (under 77) and T5 gets full prompt. Moving forward, more focus will be put on character recognition rather than art styles (although I might still incidentally get some of those while training characters) and specific poses (starting with danbooru tag poses and equivalent natural language caption for them). All images straight from the model -- no lora, or inpainting. Use Art by __________ as first tag. Photographic images use photograph of a __________, real life, __________. Can do SFW or NSFW. List of Artist Styles for Dynamic Prompt can be found at https://files.catbox.moe/w413ru.txt.
Description
FAQ
Comments (2)
Training settings -- single 4090.
{
"__version": 6,
"training_method": "FINE_TUNE",
"model_type": "STABLE_DIFFUSION_35",
"debug_mode": false,
"debug_dir": "debug",
"workspace_dir": "workspace/run",
"cache_dir": "workspace-cache/run",
"tensorboard": false,
"tensorboard_expose": false,
"validation": false,
"validate_after": 1,
"validate_after_unit": "EPOCH",
"continue_last_backup": true,
"include_train_config": "NONE",
"base_model_name": "BASEMODELHERE",
"weight_dtype": "BFLOAT_16",
"output_dtype": "FLOAT_16",
"output_model_format": "DIFFUSERS",
"output_model_destination": "models/model.safetensors",
"gradient_checkpointing": "ON",
"layer_offload_fraction": 0.0,
"force_circular_padding": false,
"concept_file_name": "training_concepts/concepts.json",
"concepts": [CONCEPTS],
"aspect_ratio_bucketing": true,
"latent_caching": true,
"clear_cache_before_training": false,
"learning_rate_scheduler": "COSINE_WITH_RESTARTS",
"custom_learning_rate_scheduler": null,
"scheduler_params": [],
"learning_rate": 2.5e-05,
"learning_rate_warmup_steps": 0,
"learning_rate_cycles": 1,
"epochs": 100,
"batch_size": 6,
"gradient_accumulation_steps": 1,
"ema": "CPU",
"ema_decay": 0.999,
"ema_update_step_interval": 5,
"dataloader_threads": 1,
"train_device": "cuda",
"temp_device": "cpu",
"train_dtype": "BFLOAT_16",
"fallback_train_dtype": "FLOAT_32",
"enable_autocast_cache": true,
"only_cache": false,
"resolution": "896,960,1024,1088,1280,1440",
"attention_mechanism": "SDP",
"align_prop": false,
"align_prop_probability": 0.1,
"align_prop_loss": "AESTHETIC",
"align_prop_weight": 0.01,
"align_prop_steps": 20,
"align_prop_truncate_steps": 0.5,
"align_prop_cfg_scale": 7.0,
"mse_strength": 1.0,
"mae_strength": 0.0,
"log_cosh_strength": 0.0,
"vb_loss_strength": 1.0,
"loss_weight_fn": "DEBIASED_ESTIMATION",
"loss_weight_strength": 5.0,
"dropout_probability": 0.0,
"loss_scaler": "NONE",
"learning_rate_scaler": "NONE",
"offset_noise_weight": 0.0,
"perturbation_noise_weight": 0.0,
"rescale_noise_scheduler_to_zero_terminal_snr": false,
"force_v_prediction": false,
"force_epsilon_prediction": false,
"min_noising_strength": 0.0,
"max_noising_strength": 0.5, #if compostions aren't training (extra arms, poses, etc) turn this up to 1.0
"timestep_distribution": "LOGIT_NORMAL",
"noising_weight": 0.0,
"noising_bias": 0.0,
"unet": {
"__version": 0,
"model_name": "",
"include": true,
"train": true,
"stop_training_after": 0,
"stop_training_after_unit": "NEVER",
"learning_rate": null,
"weight_dtype": "NONE",
"dropout_probability": 0.0,
"train_embedding": true,
"attention_mask": false
},
"prior": {
"__version": 0,
"model_name": "",
"include": true,
"train": true,
"stop_training_after": 0,
"stop_training_after_unit": "NEVER",
"learning_rate": null,
"weight_dtype": "FLOAT_32",
"dropout_probability": 0.0,
"train_embedding": true,
"attention_mask": false
},
"text_encoder": {
"__version": 0,
"model_name": "",
"include": true,
"train": true,
"stop_training_after": 30,
"stop_training_after_unit": "NEVER",
"learning_rate": 1e-08,
"weight_dtype": "NONE",
"dropout_probability": 0.0,
"train_embedding": false,
"attention_mask": false
},
"text_encoder_layer_skip": 0,
"text_encoder_2": {
"__version": 0,
"model_name": "",
"include": true,
"train": true,
"stop_training_after": 30,
"stop_training_after_unit": "NEVER",
"learning_rate": 1e-08,
"weight_dtype": "NONE",
"dropout_probability": 0.0,
"train_embedding": false,
"attention_mask": false
},
"text_encoder_2_layer_skip": 0,
"text_encoder_3": {
"__version": 0,
"model_name": "",
"include": true,
"train": false,
"stop_training_after": 30,
"stop_training_after_unit": "EPOCH",
"learning_rate": null,
"weight_dtype": "NONE",
"dropout_probability": 0.0,
"train_embedding": false,
"attention_mask": false
},
"text_encoder_3_layer_skip": 0,
"vae": {
"__version": 0,
"model_name": "",
"include": true,
"train": true,
"stop_training_after": null,
"stop_training_after_unit": "NEVER",
"learning_rate": null,
"weight_dtype": "FLOAT_32",
"dropout_probability": 0.0,
"train_embedding": true,
"attention_mask": false
},
"effnet_encoder": {
"__version": 0,
"model_name": "",
"include": true,
"train": true,
"stop_training_after": null,
"stop_training_after_unit": "NEVER",
"learning_rate": null,
"weight_dtype": "NONE",
"dropout_probability": 0.0,
"train_embedding": true,
"attention_mask": false
},
"decoder": {
"__version": 0,
"model_name": "",
"include": true,
"train": true,
"stop_training_after": null,
"stop_training_after_unit": "NEVER",
"learning_rate": null,
"weight_dtype": "NONE",
"dropout_probability": 0.0,
"train_embedding": true,
"attention_mask": false
},
"decoder_text_encoder": {
"__version": 0,
"model_name": "",
"include": true,
"train": true,
"stop_training_after": null,
"stop_training_after_unit": "NEVER",
"learning_rate": null,
"weight_dtype": "NONE",
"dropout_probability": 0.0,
"train_embedding": true,
"attention_mask": false
},
"decoder_vqgan": {
"__version": 0,
"model_name": "",
"include": true,
"train": true,
"stop_training_after": null,
"stop_training_after_unit": "NEVER",
"learning_rate": null,
"weight_dtype": "NONE",
"dropout_probability": 0.0,
"train_embedding": true,
"attention_mask": false
},
"masked_training": false,
"unmasked_probability": 0.1,
"unmasked_weight": 0.1,
"normalize_masked_area_loss": false,
"embedding_learning_rate": null,
"preserve_embedding_norm": false,
"embedding": {
"__version": 0,
"uuid": "05738880-bedd-4f76-812a-b1eaced0ac95",
"model_name": "",
"placeholder": "<embedding>",
"train": true,
"stop_training_after": null,
"stop_training_after_unit": "NEVER",
"token_count": 1,
"initial_embedding_text": "*"
},
"additional_embeddings": [],
"embedding_weight_dtype": "FLOAT_32",
"peft_type": "LORA",
"lora_model_name": "",
"lora_rank": 16,
"lora_alpha": 1.0,
"lora_decompose": false,
"lora_decompose_norm_epsilon": true,
"lora_weight_dtype": "FLOAT_32",
"lora_layers": "",
"lora_layer_preset": null,
"bundle_additional_embeddings": true,
"optimizer": {
"__version": 0,
"optimizer": "ADAFACTOR",
"adam_w_mode": false,
"alpha": null,
"amsgrad": false,
"beta1": null,
"beta2": null,
"beta3": null,
"bias_correction": false,
"block_wise": false,
"capturable": false,
"centered": false,
"clip_threshold": 1.0,
"d0": null,
"d_coef": null,
"dampening": null,
"decay_rate": -0.8,
"decouple": false,
"differentiable": false,
"eps": 1e-30,
"eps2": 0.001,
"foreach": false,
"fsdp_in_use": false,
"fused": false,
"fused_back_pass": true,
"growth_rate": null,
"initial_accumulator_value": null,
"is_paged": false,
"log_every": null,
"lr_decay": null,
"max_unorm": null,
"maximize": false,
"min_8bit_size": null,
"momentum": null,
"nesterov": false,
"no_prox": false,
"optim_bits": null,
"percentile_clipping": null,
"r": null,
"relative_step": false,
"safeguard_warmup": false,
"scale_parameter": false,
"stochastic_rounding": true,
"use_bias_correction": false,
"use_triton": false,
"warmup_init": false,
"weight_decay": 0.0,
"weight_lr_power": null,
"decoupled_decay": false,
"fixed_decay": false,
"rectify": false,
"degenerated_to_sgd": false,
"k": null,
"xi": null,
"n_sma_threshold": null,
"ams_bound": false,
"adanorm": false,
"adam_debias": false
},
"optimizer_defaults": {
"ADAFACTOR": {
"__version": 0,
"optimizer": "ADAFACTOR",
"adam_w_mode": false,
"alpha": null,
"amsgrad": false,
"beta1": null,
"beta2": null,
"beta3": null,
"bias_correction": false,
"block_wise": false,
"capturable": false,
"centered": false,
"clip_threshold": 1.0,
"d0": null,
"d_coef": null,
"dampening": null,
"decay_rate": -0.8,
"decouple": false,
"differentiable": false,
"eps": 1e-30,
"eps2": 0.001,
"foreach": false,
"fsdp_in_use": false,
"fused": false,
"fused_back_pass": true,
"growth_rate": null,
"initial_accumulator_value": null,
"is_paged": false,
"log_every": null,
"lr_decay": null,
"max_unorm": null,
"maximize": false,
"min_8bit_size": null,
"momentum": null,
"nesterov": false,
"no_prox": false,
"optim_bits": null,
"percentile_clipping": null,
"r": null,
"relative_step": false,
"safeguard_warmup": false,
"scale_parameter": false,
"stochastic_rounding": true,
"use_bias_correction": false,
"use_triton": false,
"warmup_init": false,
"weight_decay": 0.0,
"weight_lr_power": null,
"decoupled_decay": false,
"fixed_decay": false,
"rectify": false,
"degenerated_to_sgd": false,
"k": null,
"xi": null,
"n_sma_threshold": null,
"ams_bound": false,
"adanorm": false,
"adam_debias": false
}
},
"sample_definition_file_name": "training_samples/samples.json",
"samples": [SAMPLES],
"sample_after": 1,
"sample_after_unit": "EPOCH",
"sample_image_format": "JPG",
"samples_to_tensorboard": true,
"non_ema_sampling": true,
"backup_after": 45,
"backup_after_unit": "MINUTE",
"rolling_backup": true,
"rolling_backup_count": 1,
"backup_before_save": true,
"save_every": 0,
"save_every_unit": "NEVER",
"save_skip_first": 0,
"save_filename_prefix": ""
}
Excited to try this but unfortunately couldnt get it running in comfy ui.