Large Rank LoRA - Experiments towards full finetuning
What is this?
This is a collection of large rank (dimension), large dataset LoRA's which are mostly for testing full-finetuning settings with combined datasets which have been used for individual LoRA's. (see each version notes for lists)
Goal is to land on a single dataset and training settings that can
Express all the concepts well in a single checkpoint/lora
Improve the base output quality aesthetics and anatomy
Then I will use that dataset and training settings for new full-finetune models.
Who is this for?
If you enjoy any of the LoRA's datasets included, you might enjoy the combined results of these as well - in testing the individual concepts/characters are also still fairly well respresented so far. (See this post for comparisons)
These might also be useful to other creators for use in model mixing/finetuning, I've trained on base Pony for compatibility with the largest amount of merge mixes.
Datasets
Datasets included have:
Manual curating of quality and aesthetic tags:
masterpiece, best quality, low quality, very aesthetic, aesthetic
Around ~1000 images each
e.g. Fantasy Anime v.1 [Pony] version includes 3374 images from LoRA datasets:
Recommended prompt structure:
Positive prompt:
{{tags}}
score_9, score_8_up, score_7_up, score_6_up, absurdres, masterpiece, best quality, very aestheticNegative prompt:
(worst quality, low quality:1.1), score_4, score_3, score_2, score_1, error, bad anatomy, bad hands, watermark, ugly, distorted, signatureDescription
Datasets (from existing LoRA's):
*Some data was removed with the larger training in mind
Bird Girl Concepts - Characters - Monster Girl Encyclopedia - 魔物娘図鑑
Bioluminescence Concepts - Monster Girl Encyclopedia - 魔物娘図鑑
Training settings (Kohya):
{
"LoRA_type": "Standard",
"LyCORIS_preset": "full",
"adaptive_noise_scale": 0,
"additional_parameters": " --optimizer_args \"decouple=True\" \"weight_decay=0.5\" \"betas=0.9,0.99\" \"use_bias_correction=False\" --lr_scheduler_type \"CosineAnnealingLR\" --lr_scheduler_args \"T_max=30\"",
"async_upload": false,
"block_alphas": "",
"block_dims": "",
"block_lr_zero_threshold": "",
"bucket_no_upscale": true,
"bucket_reso_steps": 256,
"bypass_mode": false,
"cache_latents": true,
"cache_latents_to_disk": true,
"caption_dropout_every_n_epochs": 0,
"caption_dropout_rate": 0,
"caption_extension": ".txt",
"clip_skip": 2,
"color_aug": false,
"constrain": 0,
"conv_alpha": 1,
"conv_block_alphas": "",
"conv_block_dims": "",
"conv_dim": 1,
"dataset_config": "",
"debiased_estimation_loss": false,
"decompose_both": false,
"dim_from_weights": false,
"dora_wd": false,
"down_lr_weight": "",
"dynamo_backend": "no",
"dynamo_mode": "default",
"dynamo_use_dynamic": false,
"dynamo_use_fullgraph": false,
"enable_bucket": true,
"epoch": 30,
"extra_accelerate_launch_args": "",
"factor": -1,
"flip_aug": false,
"fp8_base": false,
"full_bf16": true,
"full_fp16": false,
"gpu_ids": "",
"gradient_accumulation_steps": 1,
"gradient_checkpointing": true,
"huber_c": 0.1,
"huber_schedule": "snr",
"huggingface_path_in_repo": "",
"huggingface_repo_id": "",
"huggingface_repo_type": "",
"huggingface_repo_visibility": "",
"huggingface_token": "",
"ip_noise_gamma": 0,
"ip_noise_gamma_random_strength": false,
"keep_tokens": 0,
"learning_rate": 1,
"log_tracker_config": "",
"log_tracker_name": "",
"log_with": "",
"logging_dir": "E:/work/LoRa_work/logging",
"loss_type": "l2",
"lr_scheduler": "cosine",
"lr_scheduler_args": "",
"lr_scheduler_num_cycles": 1,
"lr_scheduler_power": 1,
"lr_warmup": 0,
"main_process_port": 0,
"masked_loss": false,
"max_bucket_reso": 2048,
"max_data_loader_n_workers": 0,
"max_grad_norm": 1,
"max_resolution": "1024,1024",
"max_timestep": 1000,
"max_token_length": 225,
"max_train_epochs": 30,
"max_train_steps": 0,
"mem_eff_attn": false,
"metadata_author": "motimalu",
"metadata_description": "",
"metadata_license": "",
"metadata_tags": "",
"metadata_title": "",
"mid_lr_weight": "",
"min_bucket_reso": 512,
"min_snr_gamma": 5,
"min_timestep": 0,
"mixed_precision": "bf16",
"model_list": "custom",
"module_dropout": 0,
"multi_gpu": false,
"multires_noise_discount": 0.3,
"multires_noise_iterations": 6,
"network_alpha": 128,
"network_dim": 128,
"network_dropout": 0,
"network_weights": "",
"noise_offset": 0,
"noise_offset_random_strength": false,
"noise_offset_type": "Multires",
"num_cpu_threads_per_process": 2,
"num_machines": 1,
"num_processes": 1,
"optimizer": "Prodigy",
"optimizer_args": "",
"output_dir": "D:/model_output",
"output_name": "monmusu_superlora_v1",
"persistent_data_loader_workers": false,
"pretrained_model_name_or_path": "C:/workspace/webui_forge_cu121_torch21/webui/models/Stable-diffusion/ponyDiffusionV6XL_v6StartWithThisOne.safetensors",
"prior_loss_weight": 1,
"random_crop": false,
"rank_dropout": 0,
"rank_dropout_scale": false,
"reg_data_dir": "",
"rescaled": false,
"resume": "",
"resume_from_huggingface": "",
"sample_every_n_epochs": 10,
"sample_every_n_steps": 0,
"sample_prompts": "score_9, score_8_up, score_7_up, score_6_up, dragon girl, transparent tail, transparent horns, transparent wings, masterpiece, best quality, very aesthetic",
"sample_sampler": "euler_a",
"save_every_n_epochs": 30,
"save_every_n_steps": 0,
"save_last_n_steps": 0,
"save_last_n_steps_state": 0,
"save_model_as": "safetensors",
"save_precision": "bf16",
"save_state": false,
"save_state_on_train_end": false,
"save_state_to_huggingface": false,
"scale_v_pred_loss_like_noise_pred": false,
"scale_weight_norms": 0,
"sdxl": true,
"sdxl_cache_text_encoder_outputs": false,
"sdxl_no_half_vae": true,
"seed": 339491249,
"shuffle_caption": true,
"stop_text_encoder_training_pct": 0,
"text_encoder_lr": 1,
"train_batch_size": 8,
"train_data_dir": "C:/data/monmusu_datasets",
"train_norm": false,
"train_on_input": true,
"training_comment": "",
"unet_lr": 1,
"unit": 1,
"up_lr_weight": "",
"use_cp": false,
"use_scalar": false,
"use_tucker": false,
"v2": false,
"v_parameterization": false,
"v_pred_like_loss": 0,
"vae": "",
"vae_batch_size": 0,
"wandb_api_key": "",
"wandb_run_name": "",
"weighted_captions": false,
"xformers": "xformers"
}FAQ
Comments (3)
Why are people still using Masterpiece for tagging in Pony? Same with best quality and low quality? Why not simply using the default Scoring system and not training in New activation phrases?
I think it's just placebo at this point. Let it be.