Large Rank LoRA - Experiments towards full finetuning
What is this?
This is a collection of large rank (dimension), large dataset LoRA's which are mostly for testing full-finetuning settings with combined datasets which have been used for individual LoRA's. (see each version notes for lists)
Goal is to land on a single dataset and training settings that can
Express all the concepts well in a single checkpoint/lora
Improve the base output quality aesthetics and anatomy
Then I will use that dataset and training settings for new full-finetune models.
Who is this for?
If you enjoy any of the LoRA's datasets included, you might enjoy the combined results of these as well - in testing the individual concepts/characters are also still fairly well respresented so far. (See this post for comparisons)
These might also be useful to other creators for use in model mixing/finetuning, I've trained on base Pony for compatibility with the largest amount of merge mixes.
Datasets
Datasets included have:
Manual curating of quality and aesthetic tags:
masterpiece, best quality, low quality, very aesthetic, aesthetic
Around ~1000 images each
e.g. Fantasy Anime v.1 [Pony] version includes 3374 images from LoRA datasets:
Recommended prompt structure:
Positive prompt:
{{tags}}
score_9, score_8_up, score_7_up, score_6_up, absurdres, masterpiece, best quality, very aestheticNegative prompt:
(worst quality, low quality:1.1), score_4, score_3, score_2, score_1, error, bad ???????, bad hands, watermark, ugly, distorted, signatureDescription
Datasets (from existing LoRA's):
Prompts and previews from these pages can be referenced for more of what this LoRA has learned
Total 3374 images across:
- Konosuba - この素晴らしい世界に爆焔を！- Collection
- Sousou no Frieren - 葬送のフリーレン - Collection
- Delicious in Dungeon - ダンジョン飯 - Collection
- Mushoku Tensei - 無職転生 - Collection
Training settings (Kohya):
I used a batch size of 10, which was right at the limit of what I could push it to while maintaining a s/it under 3.
After epoch 20, s/it gradually increased, which resulted in a total training time of 36hrs:
epoch 21/30
steps: 70%|██████████████████████████████████▎ | 7287/10410 [7:09:19<3:03:59, 3.53s/it, avr_loss=0.0897]
epoch 22/30
steps: 73%|███████████████████████████████████▏ | 7634/10410 [12:11:19<4:25:56, 5.75s/it, avr_loss=0.0898]
epoch 23/30
steps: 77%|████████████████████████████████████▊ | 7981/10410 [16:27:01<5:00:23, 7.42s/it, avr_loss=0.0887]
epoch 24/30
steps: 80%|██████████████████████████████████████▍ | 8328/10410 [19:53:56<4:58:29, 8.60s/it, avr_loss=0.0878]
epoch 25/30
steps: 83%|████████████████████████████████████████ | 8675/10410 [22:45:11<4:33:02, 9.44s/it, avr_loss=0.0891]
epoch 26/30
steps: 87%|█████████████████████████████████████████▌ | 9022/10410 [25:32:23<3:55:45, 10.19s/it, avr_loss=0.0892]
epoch 27/30
steps: 90%|███████████████████████████████████████████▏ | 9369/10410 [28:18:39<3:08:44, 10.88s/it, avr_loss=0.0858]
epoch 28/30
steps: 93%|████████████████████████████████████████████▊ | 9716/10410 [31:04:01<2:13:08, 11.51s/it, avr_loss=0.0892]
epoch 29/30
steps: 97%|█████████████████████████████████████████████▍ | 10063/10410 [33:51:14<1:10:02, 12.11s/it, avr_loss=0.0874]
epoch 30/30
steps: 100%|█████████████████████████████████████████████████| 10410/10410 [3???1:58<00:00, 12.69s/it, avr_loss=0.0894]2Kohya Config:
I don't recommend these settings for a 24gb VRAM system (reduce to batch size of 8 at least):
{
"LoRA_type": "Standard",
"LyCORIS_preset": "full",
"adaptive_noise_scale": 0,
"additional_parameters": " --optimizer_args \"decouple=True\" \"weight_decay=0.5\" \"betas=0.9,0.99\" \"use_bias_correction=False\" --lr_scheduler_type \"CosineAnnealingLR\" --lr_scheduler_args \"T_max=30\"",
"async_upload": false,
"block_alphas": "",
"block_dims": "",
"block_lr_zero_threshold": "",
"bucket_no_upscale": true,
"bucket_reso_steps": 256,
"bypass_mode": false,
"cache_latents": true,
"cache_latents_to_disk": true,
"caption_dropout_every_n_epochs": 0,
"caption_dropout_rate": 0,
"caption_extension": ".txt",
"clip_skip": 2,
"color_aug": false,
"constrain": 0,
"conv_alpha": 1,
"conv_block_alphas": "",
"conv_block_dims": "",
"conv_dim": 1,
"dataset_config": "",
"debiased_estimation_loss": false,
"decompose_both": false,
"dim_from_weights": false,
"dora_wd": false,
"down_lr_weight": "",
"dynamo_backend": "no",
"dynamo_mode": "default",
"dynamo_use_dynamic": false,
"dynamo_use_fullgraph": false,
"enable_bucket": true,
"epoch": 30,
"extra_accelerate_launch_args": "",
"factor": -1,
"flip_aug": false,
"fp8_base": false,
"full_bf16": true,
"full_fp16": false,
"gpu_ids": "",
"gradient_accumulation_steps": 1,
"gradient_checkpointing": true,
"huber_c": 0.1,
"huber_schedule": "snr",
"huggingface_path_in_repo": "",
"huggingface_repo_id": "",
"huggingface_repo_type": "",
"huggingface_repo_visibility": "",
"huggingface_token": "",
"ip_noise_gamma": 0,
"ip_noise_gamma_random_strength": false,
"keep_tokens": 0,
"learning_rate": 1,
"log_tracker_config": "",
"log_tracker_name": "",
"log_with": "",
"logging_dir": "E:/work/LoRa_work/logging",
"loss_type": "l2",
"lr_scheduler": "cosine",
"lr_scheduler_args": "",
"lr_scheduler_num_cycles": 1,
"lr_scheduler_power": 1,
"lr_warmup": 0,
"main_process_port": 0,
"masked_loss": false,
"max_bucket_reso": 2048,
"max_data_loader_n_workers": 0,
"max_grad_norm": 1,
"max_resolution": "1024,1024",
"max_timestep": 1000,
"max_token_length": 225,
"max_train_epochs": 30,
"max_train_steps": 0,
"mem_eff_attn": false,
"metadata_author": "motimalu",
"metadata_description": "",
"metadata_license": "",
"metadata_tags": "",
"metadata_title": "",
"mid_lr_weight": "",
"min_bucket_reso": 512,
"min_snr_gamma": 5,
"min_timestep": 0,
"mixed_precision": "bf16",
"model_list": "custom",
"module_dropout": 0,
"multi_gpu": false,
"multires_noise_discount": 0.3,
"multires_noise_iterations": 6,
"network_alpha": 128,
"network_dim": 128,
"network_dropout": 0,
"network_weights": "",
"noise_offset": 0,
"noise_offset_random_strength": false,
"noise_offset_type": "Multires",
"num_cpu_threads_per_process": 2,
"num_machines": 1,
"num_processes": 1,
"optimizer": "Prodigy",
"optimizer_args": "",
"output_dir": "D:/model_output",
"output_name": "fantasy_superlora_v1",
"persistent_data_loader_workers": false,
"pretrained_model_name_or_path": "C:/workspace/webui_forge_cu121_torch21/webui/models/Stable-diffusion/ponyDiffusionV6XL_v6StartWithThisOne.safetensors",
"prior_loss_weight": 1,
"random_crop": false,
"rank_dropout": 0,
"rank_dropout_scale": false,
"reg_data_dir": "",
"rescaled": false,
"resume": "",
"resume_from_huggingface": "",
"sample_every_n_epochs": 10,
"sample_every_n_steps": 0,
"sample_prompts": "score_9, score_8_up, score_7_up, score_6_up, 1girl, frieren, white hair, twintails, green eyes, earrings, white capelet, masterpiece, best quality, very aesthetic",
"sample_sampler": "euler_a",
"save_every_n_epochs": 30,
"save_every_n_steps": 0,
"save_last_n_steps": 0,
"save_last_n_steps_state": 0,
"save_model_as": "safetensors",
"save_precision": "bf16",
"save_state": false,
"save_state_on_train_end": false,
"save_state_to_huggingface": false,
"scale_v_pred_loss_like_noise_pred": false,
"scale_weight_norms": 0,
"sdxl": true,
"sdxl_cache_text_encoder_outputs": false,
"sdxl_no_half_vae": true,
"seed": 339491249,
"shuffle_caption": false,
"stop_text_encoder_training_pct": 0,
"text_encoder_lr": 1,
"train_batch_size": 10,
"train_data_dir": "C:/data/superlora_fantasy",
"train_norm": false,
"train_on_input": true,
"training_comment": "",
"unet_lr": 1,
"unit": 1,
"up_lr_weight": "",
"use_cp": false,
"use_scalar": false,
"use_tucker": false,
"v2": false,
"v_parameterization": false,
"v_pred_like_loss": 0,
"vae": "",
"vae_batch_size": 0,
"wandb_api_key": "",
"wandb_run_name": "",
"weighted_captions": false,
"xformers": "xformers"
}