From d4c10e7d4c7753741f899fda63edb16779ade1db Mon Sep 17 00:00:00 2001 From: saftle Date: Mon, 10 Mar 2025 12:15:22 +0100 Subject: [PATCH 01/14] Created Setup.py for easy Miniconda setup --- setup.sh | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 setup.sh diff --git a/setup.sh b/setup.sh new file mode 100644 index 0000000..20b47c7 --- /dev/null +++ b/setup.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# Update system packages +sudo apt update && sudo apt install -y python3 python3-pip git wget libgl1-mesa-glx libglib2.0-0 jupyterlab + +# Set up Miniconda in your home directory +CONDA_INSTALL_DIR=$HOME/miniconda3 +wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh +chmod +x Miniconda3-latest-Linux-x86_64.sh +./Miniconda3-latest-Linux-x86_64.sh -b -p $CONDA_INSTALL_DIR +rm Miniconda3-latest-Linux-x86_64.sh + +# Add conda to current session (without affecting .bashrc yet) +export PATH="$CONDA_INSTALL_DIR/bin:$PATH" + +# Initialize conda for bash +$CONDA_INSTALL_DIR/bin/conda init bash + +# Source .bashrc for immediate use or use conda directly +eval "$($CONDA_INSTALL_DIR/bin/conda shell.bash hook)" + +# Create the conda environment +conda create -n spm python=3.10 -y + +# Define a function to run commands in the conda environment +function run_in_conda() { + conda run -n spm --no-capture-output bash -c "$*" +} + +# Clone SPM repository +git clone https://github.com/civitai/SPM +cd SPM/ + +# Install packages within the conda environment +run_in_conda "pip install -U pip" +run_in_conda "pip install torch --index-url https://download.pytorch.org/whl/cu118" +run_in_conda "pip install torchvision --index-url https://download.pytorch.org/whl/cu118" +run_in_conda "pip install xformers --index-url https://download.pytorch.org/whl/cu118" +run_in_conda "pip install huggingface_hub==0.25.0" + +# Modify requirements file to avoid conflicts +sed -i 's/^\(library==.*\)/#\1/' requirements.txt +sed 's/==[^# ]*//g' requirements.txt > requirements_nv.txt + +# Install requirements within the conda environment +run_in_conda "pip install -r ./requirements_nv.txt" From 5944c0ea7cff11f62e14f34d5a4b56530bb93b50 Mon Sep 17 00:00:00 2001 From: saftle Date: Mon, 10 Mar 2025 12:18:46 +0100 Subject: [PATCH 02/14] Added SD1.5 Child Removal Jupyter Notebook --- sd15_child_removal.ipynb | 35221 +++++++++++++++++++++++++++++++++++++ 1 file changed, 35221 insertions(+) create mode 100644 sd15_child_removal.ipynb diff --git a/sd15_child_removal.ipynb b/sd15_child_removal.ipynb new file mode 100644 index 0000000..ae4bf26 --- /dev/null +++ b/sd15_child_removal.ipynb @@ -0,0 +1,35221 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "view-in-github" + }, + "source": [ + "\"Open" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "id": "_U-cYMtEVY2K" + }, + "source": [ + "## Create Configs" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "id": "XBgbejdDV2iq" + }, + "source": [ + "### Step 1: Choose Base Model\n", + "\n", + "Examples of `pretrained_sd_model`:\n", + "- SD v1.4: `CompVis/stable-diffusion-v1-4`\n", + "- SD v1.5: `runwayml/stable-diffusion-v1-5`\n", + "- Dreamshaper v8: `stablediffusionapi/dreamshaper-v8`\n", + "- SD v2.1: `stabilityai/stable-diffusion-2-1-base`\n", + "- WD1.5 beta3: `Birchlabs/wd-1-5-beta3-unofficial`\n", + "- SDXL: `stabilityai/stable-diffusion-xl-base-1.0`\n", + "- Juggernaut v6: `RunDiffusion/Juggernaut-XL-v6`\n", + "- PonyXL: `stablediffusionapi/pony-diffusion-v6-xl`\n", + "\n", + "If base model is v2.x, set `is_v2_model` to `true`." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "pretrained_sd_model = r\"/media/alex/d5f7850c-8e1d-4f90-acd3-58912cc12a4a/ComfyUI/models/checkpoints/URPMv1.5.4.safetensors\" #@param {type: \"string\"}\n", + "is_v2_model = \"false\" #@param [\"true\", \"false\"]\n", + "is_v_prediction_model = \"false\" #@param [\"true\", \"false\"]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Step 2: Choose Concept\n", + "\n", + "- `target_concept`: Targeted concept for erasing\n", + "- `surrogate_concept`: Surrogate concept for defining model generation after erasure, empty string by default" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# If you want to train multiple concepts at once in a single SPM, set this to True\n", + "multiple_concepts = True #@param {type: \"boolean\"}\n", + "\n", + "target_concept = '' #@param {type: \"string\"}\n", + "surrogate_concept = '' #@param {type: \"string\"}\n", + "\n", + "# If multiple_concepts is True, the following fields will be used instead\n", + "target_concepts = ['teen','child','kid','loli','toddler','preteen','baby','newborn','girl','boy','young girl','young boy','year old','years old','little boy','little girl'] #@param {type: \"string\"}\n", + "surrogate_concepts = ['adult','adult','adult','woman','adult','adult','adult','adult','woman','man','woman','man','20 year old','20 years old','man','woman'] #@param {type: \"string\"}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Step 3: SPM Settings\n", + "\n", + "- `mode`: `erase_with_la` or `erase` (for ablation)\n", + "- `dim`: default to 1, other values for ablation\n", + "- `sampling_batch_size`: indicates how many latent anchors are sampled for each iteration, default to 4, can be reduced if there's not enough VRAM\n", + "- `la_strength`: indicates the latent anchoring loss strength that balancing the erasure and preservation, default to 1000 for SD v1.4 models, should be further tuned for other base models for better performance\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "mode = 'erase_with_la' #@param [\"erase_with_la\", \"erase\"]\n", + "dim = 2 #@param {type: \"number\"}\n", + "sampling_batch_size = 4 #@param {type: \"number\"}\n", + "la_strength = 250 #@param {type: \"number\"}\n", + "erasing_scale = 1.5 #@param {type: \"number\"}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Step 4: Training Settings\n", + "\n", + "There are two parts for training settings, SD settings and optimization settings.\n", + "Notice that `resolution` is set to 512 for SD v1.x, 768 for SD v2.x, and 1024 for SDXL." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "resolution = 512 #@param [512, 640, 768, 896, 960, 1024]\n", + "max_denoising_steps = 20 #@param {type: \"number\"}\n", + "dynamic_resolution = \"true\" #@param [\"true\", \"false\"]\n", + "clip_skip = 1 #@param [1, 2]\n", + "\n", + "batch_size = 4 #@param {type: \"number\"}\n", + "iterations = 6000 #@param {type: \"number\"}\n", + "lr = 15e-5 #@param {type: \"number\"}\n", + "optimizer = \"AdamW8bit\" #@param {type: \"string\"}\n", + "lr_scheduler = \"cosine_with_restarts\" #@param {type: \"string\"}\n", + "lr_warmup_steps = 300 #@param {type: \"number\"}\n", + "lr_scheduler_num_cycles = 3 #@param {type: \"number\"}\n", + "save_per_steps = 750 #@param {type: \"number\"}\n", + "precision = \"bfloat16\" #@param [\"float32\", \"float16\", \"bfloat16\"]\n", + "verbose = \"false\" #@param [\"true\", \"false\"]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Step 5: (Optional) Tracking Training Details with WandB\n", + "\n", + "You can setup your wandb token to track the training details, including training statistics (e.g. losses, learning rates) and visualizations.\n", + "Your wandb token can be retrieved from https://wandb.ai/authorize ." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "#wandb_token = \"\" #@param {type: \"string\"}\n", + "\n", + "#prompts_to_visualize = [\"cat\", \"dog\"] #@param {type: \"string\"}\n", + "#generate_num = 2 #@param {type: \"number\"}\n", + "\n", + "## track target & surrogate by default\n", + "#prompts_to_visualize = [target_concept, surrogate_concept] + prompts_to_visualize\n", + "\n", + "## login with your wandb token\n", + "#if wandb_token != \"\": \n", + "# !wandb login {wandb_token}\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Step 6: Generate Config Files\n", + "\n", + "Run the following code block and the config files are automatically generated." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "cellView": "form", + "id": "a1pK61X_VY2L" + }, + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'generate_num' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[7], line 51\u001b[0m\n\u001b[1;32m 10\u001b[0m config_file_path \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mconfigs/\u001b[39m\u001b[38;5;132;01m{\u001b[39;00msave_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m/config.yaml\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 11\u001b[0m prompts_file_path \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mconfigs/\u001b[39m\u001b[38;5;132;01m{\u001b[39;00msave_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m/prompt.yaml\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 14\u001b[0m config_file_content \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\"\"\u001b[39m\n\u001b[1;32m 15\u001b[0m \u001b[38;5;124mprompts_file: \u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mprompts_file_path\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 16\u001b[0m \n\u001b[1;32m 17\u001b[0m \u001b[38;5;124mpretrained_model:\u001b[39m\n\u001b[1;32m 18\u001b[0m \u001b[38;5;124m name_or_path: \u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mpretrained_sd_model\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 19\u001b[0m \u001b[38;5;124m v2: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mis_v2_model\u001b[38;5;132;01m}\u001b[39;00m\n\u001b[1;32m 20\u001b[0m \u001b[38;5;124m v_pred: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mis_v_prediction_model\u001b[38;5;132;01m}\u001b[39;00m\n\u001b[1;32m 21\u001b[0m \u001b[38;5;124m clip_skip: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mclip_skip\u001b[38;5;132;01m}\u001b[39;00m\n\u001b[1;32m 22\u001b[0m \n\u001b[1;32m 23\u001b[0m \u001b[38;5;124mnetwork:\u001b[39m\n\u001b[1;32m 24\u001b[0m \u001b[38;5;124m rank: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mdim\u001b[38;5;132;01m}\u001b[39;00m\n\u001b[1;32m 25\u001b[0m \u001b[38;5;124m alpha: 1.0\u001b[39m\n\u001b[1;32m 26\u001b[0m \n\u001b[1;32m 27\u001b[0m \u001b[38;5;124mtrain:\u001b[39m\n\u001b[1;32m 28\u001b[0m \u001b[38;5;124m precision: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mprecision\u001b[38;5;132;01m}\u001b[39;00m\n\u001b[1;32m 29\u001b[0m \u001b[38;5;124m noise_scheduler: \u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mddim\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 30\u001b[0m \u001b[38;5;124m iterations: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00miterations\u001b[38;5;132;01m}\u001b[39;00m\n\u001b[1;32m 31\u001b[0m \u001b[38;5;124m batch_size: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mbatch_size\u001b[38;5;132;01m}\u001b[39;00m\n\u001b[1;32m 32\u001b[0m \u001b[38;5;124m lr: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mlr\u001b[38;5;132;01m}\u001b[39;00m\n\u001b[1;32m 33\u001b[0m \u001b[38;5;124m unet_lr: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mlr\u001b[38;5;132;01m}\u001b[39;00m\n\u001b[1;32m 34\u001b[0m \u001b[38;5;124m text_encoder_lr: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;241m0.5\u001b[39m\u001b[38;5;250m \u001b[39m\u001b[38;5;241m*\u001b[39m\u001b[38;5;250m \u001b[39mlr\u001b[38;5;132;01m}\u001b[39;00m\n\u001b[1;32m 35\u001b[0m \u001b[38;5;124m optimizer_type: \u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00moptimizer\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 36\u001b[0m \u001b[38;5;124m lr_scheduler: \u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mlr_scheduler\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 37\u001b[0m \u001b[38;5;124m lr_warmup_steps: 500\u001b[39m\n\u001b[1;32m 38\u001b[0m \u001b[38;5;124m lr_scheduler_num_cycles: 3\u001b[39m\n\u001b[1;32m 39\u001b[0m \u001b[38;5;124m max_denoising_steps: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmax_denoising_steps\u001b[38;5;132;01m}\u001b[39;00m\n\u001b[1;32m 40\u001b[0m \n\u001b[1;32m 41\u001b[0m \u001b[38;5;124msave:\u001b[39m\n\u001b[1;32m 42\u001b[0m \u001b[38;5;124m name: \u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00msave_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 43\u001b[0m \u001b[38;5;124m path: \u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutput/\u001b[39m\u001b[38;5;132;01m{\u001b[39;00msave_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 44\u001b[0m \u001b[38;5;124m per_steps: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00msave_per_steps\u001b[38;5;132;01m}\u001b[39;00m\n\u001b[1;32m 45\u001b[0m \u001b[38;5;124m precision: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mprecision\u001b[38;5;132;01m}\u001b[39;00m\n\u001b[1;32m 46\u001b[0m \n\u001b[1;32m 47\u001b[0m \u001b[38;5;124mlogging:\u001b[39m\n\u001b[1;32m 48\u001b[0m \u001b[38;5;124m use_wandb: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfalse\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\n\u001b[1;32m 49\u001b[0m \u001b[38;5;124m interval: 500\u001b[39m\n\u001b[1;32m 50\u001b[0m \u001b[38;5;124m seed: 0\u001b[39m\n\u001b[0;32m---> 51\u001b[0m \u001b[38;5;124m generate_num: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[43mgenerate_num\u001b[49m\u001b[38;5;132;01m}\u001b[39;00m\n\u001b[1;32m 52\u001b[0m \u001b[38;5;124m run_name: \u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mrun_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 53\u001b[0m \u001b[38;5;124m verbose: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mverbose\u001b[38;5;132;01m}\u001b[39;00m\n\u001b[1;32m 54\u001b[0m \u001b[38;5;124m prompts: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mprompts_to_visualize\u001b[38;5;132;01m}\u001b[39;00m\n\u001b[1;32m 55\u001b[0m \n\u001b[1;32m 56\u001b[0m \u001b[38;5;124mother:\u001b[39m\n\u001b[1;32m 57\u001b[0m \u001b[38;5;124m use_xformers: true\u001b[39m\n\u001b[1;32m 58\u001b[0m \u001b[38;5;124m\"\"\"\u001b[39m\n\u001b[1;32m 60\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mos\u001b[39;00m\n\u001b[1;32m 61\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mexists(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m./configs/\u001b[39m\u001b[38;5;132;01m{\u001b[39;00msave_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m):\n", + "\u001b[0;31mNameError\u001b[0m: name 'generate_num' is not defined" + ] + } + ], + "source": [ + "# you can custom these strings to distiguish your different exps\n", + "exp_name = target_concept.replace(\" \", \"_\") if not multiple_concepts else \"multi_concepts\"\n", + "if multiple_concepts:\n", + " exp_name += \"_to_multiple_concepts\"\n", + "elif surrogate_concept:\n", + " exp_name += f\"_to_{surrogate_concept.replace(' ', '_')}\"\n", + "save_name = f\"{exp_name}\"\n", + "run_name = f\"{exp_name}\"\n", + "\n", + "config_file_path = f\"configs/{save_name}/config.yaml\"\n", + "prompts_file_path = f\"configs/{save_name}/prompt.yaml\"\n", + "\n", + "\n", + "config_file_content = f\"\"\"\n", + "prompts_file: \"{prompts_file_path}\"\n", + "\n", + "pretrained_model:\n", + " name_or_path: \"{pretrained_sd_model}\"\n", + " v2: {is_v2_model}\n", + " v_pred: {is_v_prediction_model}\n", + " clip_skip: {clip_skip}\n", + "\n", + "network:\n", + " rank: {dim}\n", + " alpha: 1.0\n", + "\n", + "train:\n", + " precision: {precision}\n", + " noise_scheduler: \"ddim\"\n", + " iterations: {iterations}\n", + " batch_size: {batch_size}\n", + " lr: {lr}\n", + " unet_lr: {lr}\n", + " text_encoder_lr: {0.5 * lr}\n", + " optimizer_type: \"{optimizer}\"\n", + " lr_scheduler: \"{lr_scheduler}\"\n", + " lr_warmup_steps: 500\n", + " lr_scheduler_num_cycles: 3\n", + " max_denoising_steps: {max_denoising_steps}\n", + "\n", + "save:\n", + " name: \"{save_name}\"\n", + " path: \"output/{save_name}\"\n", + " per_steps: {save_per_steps}\n", + " precision: {precision}\n", + "\n", + "logging:\n", + " use_wandb: {\"false\"}\n", + " interval: 500\n", + " seed: 0\n", + " generate_num: {generate_num}\n", + " run_name: \"{run_name}\"\n", + " verbose: {verbose}\n", + " prompts: {prompts_to_visualize}\n", + "\n", + "other:\n", + " use_xformers: true\n", + "\"\"\"\n", + "\n", + "import os\n", + "if not os.path.exists(f\"./configs/{save_name}\"):\n", + " os.makedirs(f\"./configs/{save_name}\")\n", + "\n", + "with open(config_file_path, \"w\") as f:\n", + " f.write(config_file_content)\n", + "\n", + "if multiple_concepts:\n", + " prompts_file_content = \"\" \n", + "\n", + " for target_concept, surrogate_concept in zip(target_concepts, surrogate_concepts):\n", + " prompts_file_content += f\"\"\"\n", + "- target: \"{target_concept}\"\n", + " positive: \"{target_concept}\"\n", + " unconditional: \"\"\n", + " neutral: \"{surrogate_concept}\"\n", + " action: \"{mode}\"\n", + " guidance_scale: \"{erasing_scale}\"\n", + " resolution: {resolution}\n", + " batch_size: {batch_size}\n", + " dynamic_resolution: {dynamic_resolution}\n", + " la_strength: {la_strength}\n", + " sampling_batch_size: {sampling_batch_size}\n", + " \"\"\"\n", + "else:\n", + " prompts_file_content = f\"\"\"\n", + "- target: \"{target_concept}\"\n", + " positive: \"{target_concept}\"\n", + " unconditional: \"\"\n", + " neutral: \"{surrogate_concept}\"\n", + " action: \"{mode}\"\n", + " guidance_scale: \"{erasing_scale}\"\n", + " resolution: {resolution}\n", + " batch_size: {batch_size}\n", + " dynamic_resolution: {dynamic_resolution}\n", + " la_strength: {la_strength}\n", + " sampling_batch_size: {sampling_batch_size}\n", + " \"\"\"\n", + "\n", + "with open(prompts_file_path, \"w\") as f:\n", + " f.write(prompts_file_content)\n" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "id": "wb9aX9BBVY2L" + }, + "source": [ + "## Start Training" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "id": "WchSOvSHVY2L" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/home/alex/SPM/./train_spm.py:41: PydanticDeprecatedSince20: The `json` method is deprecated; use `model_dump_json` instead. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.10/migration/\n", + " \"prompts\": \",\".join([prompt.json() for prompt in prompts]),\n", + "/home/alex/SPM/./train_spm.py:42: PydanticDeprecatedSince20: The `json` method is deprecated; use `model_dump_json` instead. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.10/migration/\n", + " \"config\": config.json(),\n", + "Loading local checkpoint from /media/alex/d5f7850c-8e1d-4f90-acd3-58912cc12a4a/ComfyUI/models/checkpoints/URPMv1.5.4.safetensors\n", + "Fetching 11 files: 100%|████████████████████| 11/11 [00:00<00:00, 279620.27it/s]\n", + "Some weights of the model checkpoint were not used when initializing CLIPTextModel: \n", + " ['text_model.embeddings.position_ids']\n", + "In this conversion only the non-EMA weights are extracted. If you want to instead extract the EMA weights (usually better for inference), please make sure to add the `--extract_ema` flag.\n", + "Loading pipeline components...: 100%|█████████████| 6/6 [00:00<00:00, 47.50it/s]\n", + "You have disabled the safety checker for by passing `safety_checker=None`. Ensure that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered results in services or applications open to the public. Both the diffusers team and Hugging Face strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling it only for use-cases that involve analyzing network behavior or auditing its results. For more information, please have a look at https://github.com/huggingface/diffusers/pull/254 .\n", + "lora_unet_down_blocks_0_attentions_0_proj_in\n", + "lora_unet_down_blocks_0_attentions_0_transformer_blocks_0_attn1_to_q\n", + "lora_unet_down_blocks_0_attentions_0_transformer_blocks_0_attn1_to_k\n", + "lora_unet_down_blocks_0_attentions_0_transformer_blocks_0_attn1_to_v\n", + "lora_unet_down_blocks_0_attentions_0_transformer_blocks_0_attn1_to_out_0\n", + "lora_unet_down_blocks_0_attentions_0_transformer_blocks_0_attn2_to_q\n", + "lora_unet_down_blocks_0_attentions_0_transformer_blocks_0_attn2_to_k\n", + "lora_unet_down_blocks_0_attentions_0_transformer_blocks_0_attn2_to_v\n", + "lora_unet_down_blocks_0_attentions_0_transformer_blocks_0_attn2_to_out_0\n", + "lora_unet_down_blocks_0_attentions_0_transformer_blocks_0_ff_net_0_proj\n", + "lora_unet_down_blocks_0_attentions_0_transformer_blocks_0_ff_net_2\n", + "lora_unet_down_blocks_0_attentions_0_proj_out\n", + "lora_unet_down_blocks_0_attentions_1_proj_in\n", + "lora_unet_down_blocks_0_attentions_1_transformer_blocks_0_attn1_to_q\n", + "lora_unet_down_blocks_0_attentions_1_transformer_blocks_0_attn1_to_k\n", + "lora_unet_down_blocks_0_attentions_1_transformer_blocks_0_attn1_to_v\n", + "lora_unet_down_blocks_0_attentions_1_transformer_blocks_0_attn1_to_out_0\n", + "lora_unet_down_blocks_0_attentions_1_transformer_blocks_0_attn2_to_q\n", + "lora_unet_down_blocks_0_attentions_1_transformer_blocks_0_attn2_to_k\n", + "lora_unet_down_blocks_0_attentions_1_transformer_blocks_0_attn2_to_v\n", + "lora_unet_down_blocks_0_attentions_1_transformer_blocks_0_attn2_to_out_0\n", + "lora_unet_down_blocks_0_attentions_1_transformer_blocks_0_ff_net_0_proj\n", + "lora_unet_down_blocks_0_attentions_1_transformer_blocks_0_ff_net_2\n", + "lora_unet_down_blocks_0_attentions_1_proj_out\n", + "lora_unet_down_blocks_1_attentions_0_proj_in\n", + "lora_unet_down_blocks_1_attentions_0_transformer_blocks_0_attn1_to_q\n", + "lora_unet_down_blocks_1_attentions_0_transformer_blocks_0_attn1_to_k\n", + "lora_unet_down_blocks_1_attentions_0_transformer_blocks_0_attn1_to_v\n", + "lora_unet_down_blocks_1_attentions_0_transformer_blocks_0_attn1_to_out_0\n", + "lora_unet_down_blocks_1_attentions_0_transformer_blocks_0_attn2_to_q\n", + "lora_unet_down_blocks_1_attentions_0_transformer_blocks_0_attn2_to_k\n", + "lora_unet_down_blocks_1_attentions_0_transformer_blocks_0_attn2_to_v\n", + "lora_unet_down_blocks_1_attentions_0_transformer_blocks_0_attn2_to_out_0\n", + "lora_unet_down_blocks_1_attentions_0_transformer_blocks_0_ff_net_0_proj\n", + "lora_unet_down_blocks_1_attentions_0_transformer_blocks_0_ff_net_2\n", + "lora_unet_down_blocks_1_attentions_0_proj_out\n", + "lora_unet_down_blocks_1_attentions_1_proj_in\n", + "lora_unet_down_blocks_1_attentions_1_transformer_blocks_0_attn1_to_q\n", + "lora_unet_down_blocks_1_attentions_1_transformer_blocks_0_attn1_to_k\n", + "lora_unet_down_blocks_1_attentions_1_transformer_blocks_0_attn1_to_v\n", + "lora_unet_down_blocks_1_attentions_1_transformer_blocks_0_attn1_to_out_0\n", + "lora_unet_down_blocks_1_attentions_1_transformer_blocks_0_attn2_to_q\n", + "lora_unet_down_blocks_1_attentions_1_transformer_blocks_0_attn2_to_k\n", + "lora_unet_down_blocks_1_attentions_1_transformer_blocks_0_attn2_to_v\n", + "lora_unet_down_blocks_1_attentions_1_transformer_blocks_0_attn2_to_out_0\n", + "lora_unet_down_blocks_1_attentions_1_transformer_blocks_0_ff_net_0_proj\n", + "lora_unet_down_blocks_1_attentions_1_transformer_blocks_0_ff_net_2\n", + "lora_unet_down_blocks_1_attentions_1_proj_out\n", + "lora_unet_down_blocks_2_attentions_0_proj_in\n", + "lora_unet_down_blocks_2_attentions_0_transformer_blocks_0_attn1_to_q\n", + "lora_unet_down_blocks_2_attentions_0_transformer_blocks_0_attn1_to_k\n", + "lora_unet_down_blocks_2_attentions_0_transformer_blocks_0_attn1_to_v\n", + "lora_unet_down_blocks_2_attentions_0_transformer_blocks_0_attn1_to_out_0\n", + "lora_unet_down_blocks_2_attentions_0_transformer_blocks_0_attn2_to_q\n", + "lora_unet_down_blocks_2_attentions_0_transformer_blocks_0_attn2_to_k\n", + "lora_unet_down_blocks_2_attentions_0_transformer_blocks_0_attn2_to_v\n", + "lora_unet_down_blocks_2_attentions_0_transformer_blocks_0_attn2_to_out_0\n", + "lora_unet_down_blocks_2_attentions_0_transformer_blocks_0_ff_net_0_proj\n", + "lora_unet_down_blocks_2_attentions_0_transformer_blocks_0_ff_net_2\n", + "lora_unet_down_blocks_2_attentions_0_proj_out\n", + "lora_unet_down_blocks_2_attentions_1_proj_in\n", + "lora_unet_down_blocks_2_attentions_1_transformer_blocks_0_attn1_to_q\n", + "lora_unet_down_blocks_2_attentions_1_transformer_blocks_0_attn1_to_k\n", + "lora_unet_down_blocks_2_attentions_1_transformer_blocks_0_attn1_to_v\n", + "lora_unet_down_blocks_2_attentions_1_transformer_blocks_0_attn1_to_out_0\n", + "lora_unet_down_blocks_2_attentions_1_transformer_blocks_0_attn2_to_q\n", + "lora_unet_down_blocks_2_attentions_1_transformer_blocks_0_attn2_to_k\n", + "lora_unet_down_blocks_2_attentions_1_transformer_blocks_0_attn2_to_v\n", + "lora_unet_down_blocks_2_attentions_1_transformer_blocks_0_attn2_to_out_0\n", + "lora_unet_down_blocks_2_attentions_1_transformer_blocks_0_ff_net_0_proj\n", + "lora_unet_down_blocks_2_attentions_1_transformer_blocks_0_ff_net_2\n", + "lora_unet_down_blocks_2_attentions_1_proj_out\n", + "lora_unet_up_blocks_1_attentions_0_proj_in\n", + "lora_unet_up_blocks_1_attentions_0_transformer_blocks_0_attn1_to_q\n", + "lora_unet_up_blocks_1_attentions_0_transformer_blocks_0_attn1_to_k\n", + "lora_unet_up_blocks_1_attentions_0_transformer_blocks_0_attn1_to_v\n", + "lora_unet_up_blocks_1_attentions_0_transformer_blocks_0_attn1_to_out_0\n", + "lora_unet_up_blocks_1_attentions_0_transformer_blocks_0_attn2_to_q\n", + "lora_unet_up_blocks_1_attentions_0_transformer_blocks_0_attn2_to_k\n", + "lora_unet_up_blocks_1_attentions_0_transformer_blocks_0_attn2_to_v\n", + "lora_unet_up_blocks_1_attentions_0_transformer_blocks_0_attn2_to_out_0\n", + "lora_unet_up_blocks_1_attentions_0_transformer_blocks_0_ff_net_0_proj\n", + "lora_unet_up_blocks_1_attentions_0_transformer_blocks_0_ff_net_2\n", + "lora_unet_up_blocks_1_attentions_0_proj_out\n", + "lora_unet_up_blocks_1_attentions_1_proj_in\n", + "lora_unet_up_blocks_1_attentions_1_transformer_blocks_0_attn1_to_q\n", + "lora_unet_up_blocks_1_attentions_1_transformer_blocks_0_attn1_to_k\n", + "lora_unet_up_blocks_1_attentions_1_transformer_blocks_0_attn1_to_v\n", + "lora_unet_up_blocks_1_attentions_1_transformer_blocks_0_attn1_to_out_0\n", + "lora_unet_up_blocks_1_attentions_1_transformer_blocks_0_attn2_to_q\n", + "lora_unet_up_blocks_1_attentions_1_transformer_blocks_0_attn2_to_k\n", + "lora_unet_up_blocks_1_attentions_1_transformer_blocks_0_attn2_to_v\n", + "lora_unet_up_blocks_1_attentions_1_transformer_blocks_0_attn2_to_out_0\n", + "lora_unet_up_blocks_1_attentions_1_transformer_blocks_0_ff_net_0_proj\n", + "lora_unet_up_blocks_1_attentions_1_transformer_blocks_0_ff_net_2\n", + "lora_unet_up_blocks_1_attentions_1_proj_out\n", + "lora_unet_up_blocks_1_attentions_2_proj_in\n", + "lora_unet_up_blocks_1_attentions_2_transformer_blocks_0_attn1_to_q\n", + "lora_unet_up_blocks_1_attentions_2_transformer_blocks_0_attn1_to_k\n", + "lora_unet_up_blocks_1_attentions_2_transformer_blocks_0_attn1_to_v\n", + "lora_unet_up_blocks_1_attentions_2_transformer_blocks_0_attn1_to_out_0\n", + "lora_unet_up_blocks_1_attentions_2_transformer_blocks_0_attn2_to_q\n", + "lora_unet_up_blocks_1_attentions_2_transformer_blocks_0_attn2_to_k\n", + "lora_unet_up_blocks_1_attentions_2_transformer_blocks_0_attn2_to_v\n", + "lora_unet_up_blocks_1_attentions_2_transformer_blocks_0_attn2_to_out_0\n", + "lora_unet_up_blocks_1_attentions_2_transformer_blocks_0_ff_net_0_proj\n", + "lora_unet_up_blocks_1_attentions_2_transformer_blocks_0_ff_net_2\n", + "lora_unet_up_blocks_1_attentions_2_proj_out\n", + "lora_unet_up_blocks_2_attentions_0_proj_in\n", + "lora_unet_up_blocks_2_attentions_0_transformer_blocks_0_attn1_to_q\n", + "lora_unet_up_blocks_2_attentions_0_transformer_blocks_0_attn1_to_k\n", + "lora_unet_up_blocks_2_attentions_0_transformer_blocks_0_attn1_to_v\n", + "lora_unet_up_blocks_2_attentions_0_transformer_blocks_0_attn1_to_out_0\n", + "lora_unet_up_blocks_2_attentions_0_transformer_blocks_0_attn2_to_q\n", + "lora_unet_up_blocks_2_attentions_0_transformer_blocks_0_attn2_to_k\n", + "lora_unet_up_blocks_2_attentions_0_transformer_blocks_0_attn2_to_v\n", + "lora_unet_up_blocks_2_attentions_0_transformer_blocks_0_attn2_to_out_0\n", + "lora_unet_up_blocks_2_attentions_0_transformer_blocks_0_ff_net_0_proj\n", + "lora_unet_up_blocks_2_attentions_0_transformer_blocks_0_ff_net_2\n", + "lora_unet_up_blocks_2_attentions_0_proj_out\n", + "lora_unet_up_blocks_2_attentions_1_proj_in\n", + "lora_unet_up_blocks_2_attentions_1_transformer_blocks_0_attn1_to_q\n", + "lora_unet_up_blocks_2_attentions_1_transformer_blocks_0_attn1_to_k\n", + "lora_unet_up_blocks_2_attentions_1_transformer_blocks_0_attn1_to_v\n", + "lora_unet_up_blocks_2_attentions_1_transformer_blocks_0_attn1_to_out_0\n", + "lora_unet_up_blocks_2_attentions_1_transformer_blocks_0_attn2_to_q\n", + "lora_unet_up_blocks_2_attentions_1_transformer_blocks_0_attn2_to_k\n", + "lora_unet_up_blocks_2_attentions_1_transformer_blocks_0_attn2_to_v\n", + "lora_unet_up_blocks_2_attentions_1_transformer_blocks_0_attn2_to_out_0\n", + "lora_unet_up_blocks_2_attentions_1_transformer_blocks_0_ff_net_0_proj\n", + "lora_unet_up_blocks_2_attentions_1_transformer_blocks_0_ff_net_2\n", + "lora_unet_up_blocks_2_attentions_1_proj_out\n", + "lora_unet_up_blocks_2_attentions_2_proj_in\n", + "lora_unet_up_blocks_2_attentions_2_transformer_blocks_0_attn1_to_q\n", + "lora_unet_up_blocks_2_attentions_2_transformer_blocks_0_attn1_to_k\n", + "lora_unet_up_blocks_2_attentions_2_transformer_blocks_0_attn1_to_v\n", + "lora_unet_up_blocks_2_attentions_2_transformer_blocks_0_attn1_to_out_0\n", + "lora_unet_up_blocks_2_attentions_2_transformer_blocks_0_attn2_to_q\n", + "lora_unet_up_blocks_2_attentions_2_transformer_blocks_0_attn2_to_k\n", + "lora_unet_up_blocks_2_attentions_2_transformer_blocks_0_attn2_to_v\n", + "lora_unet_up_blocks_2_attentions_2_transformer_blocks_0_attn2_to_out_0\n", + "lora_unet_up_blocks_2_attentions_2_transformer_blocks_0_ff_net_0_proj\n", + "lora_unet_up_blocks_2_attentions_2_transformer_blocks_0_ff_net_2\n", + "lora_unet_up_blocks_2_attentions_2_proj_out\n", + "lora_unet_up_blocks_3_attentions_0_proj_in\n", + "lora_unet_up_blocks_3_attentions_0_transformer_blocks_0_attn1_to_q\n", + "lora_unet_up_blocks_3_attentions_0_transformer_blocks_0_attn1_to_k\n", + "lora_unet_up_blocks_3_attentions_0_transformer_blocks_0_attn1_to_v\n", + "lora_unet_up_blocks_3_attentions_0_transformer_blocks_0_attn1_to_out_0\n", + "lora_unet_up_blocks_3_attentions_0_transformer_blocks_0_attn2_to_q\n", + "lora_unet_up_blocks_3_attentions_0_transformer_blocks_0_attn2_to_k\n", + "lora_unet_up_blocks_3_attentions_0_transformer_blocks_0_attn2_to_v\n", + "lora_unet_up_blocks_3_attentions_0_transformer_blocks_0_attn2_to_out_0\n", + "lora_unet_up_blocks_3_attentions_0_transformer_blocks_0_ff_net_0_proj\n", + "lora_unet_up_blocks_3_attentions_0_transformer_blocks_0_ff_net_2\n", + "lora_unet_up_blocks_3_attentions_0_proj_out\n", + "lora_unet_up_blocks_3_attentions_1_proj_in\n", + "lora_unet_up_blocks_3_attentions_1_transformer_blocks_0_attn1_to_q\n", + "lora_unet_up_blocks_3_attentions_1_transformer_blocks_0_attn1_to_k\n", + "lora_unet_up_blocks_3_attentions_1_transformer_blocks_0_attn1_to_v\n", + "lora_unet_up_blocks_3_attentions_1_transformer_blocks_0_attn1_to_out_0\n", + "lora_unet_up_blocks_3_attentions_1_transformer_blocks_0_attn2_to_q\n", + "lora_unet_up_blocks_3_attentions_1_transformer_blocks_0_attn2_to_k\n", + "lora_unet_up_blocks_3_attentions_1_transformer_blocks_0_attn2_to_v\n", + "lora_unet_up_blocks_3_attentions_1_transformer_blocks_0_attn2_to_out_0\n", + "lora_unet_up_blocks_3_attentions_1_transformer_blocks_0_ff_net_0_proj\n", + "lora_unet_up_blocks_3_attentions_1_transformer_blocks_0_ff_net_2\n", + "lora_unet_up_blocks_3_attentions_1_proj_out\n", + "lora_unet_up_blocks_3_attentions_2_proj_in\n", + "lora_unet_up_blocks_3_attentions_2_transformer_blocks_0_attn1_to_q\n", + "lora_unet_up_blocks_3_attentions_2_transformer_blocks_0_attn1_to_k\n", + "lora_unet_up_blocks_3_attentions_2_transformer_blocks_0_attn1_to_v\n", + "lora_unet_up_blocks_3_attentions_2_transformer_blocks_0_attn1_to_out_0\n", + "lora_unet_up_blocks_3_attentions_2_transformer_blocks_0_attn2_to_q\n", + "lora_unet_up_blocks_3_attentions_2_transformer_blocks_0_attn2_to_k\n", + "lora_unet_up_blocks_3_attentions_2_transformer_blocks_0_attn2_to_v\n", + "lora_unet_up_blocks_3_attentions_2_transformer_blocks_0_attn2_to_out_0\n", + "lora_unet_up_blocks_3_attentions_2_transformer_blocks_0_ff_net_0_proj\n", + "lora_unet_up_blocks_3_attentions_2_transformer_blocks_0_ff_net_2\n", + "lora_unet_up_blocks_3_attentions_2_proj_out\n", + "lora_unet_mid_block_attentions_0_proj_in\n", + "lora_unet_mid_block_attentions_0_transformer_blocks_0_attn1_to_q\n", + "lora_unet_mid_block_attentions_0_transformer_blocks_0_attn1_to_k\n", + "lora_unet_mid_block_attentions_0_transformer_blocks_0_attn1_to_v\n", + "lora_unet_mid_block_attentions_0_transformer_blocks_0_attn1_to_out_0\n", + "lora_unet_mid_block_attentions_0_transformer_blocks_0_attn2_to_q\n", + "lora_unet_mid_block_attentions_0_transformer_blocks_0_attn2_to_k\n", + "lora_unet_mid_block_attentions_0_transformer_blocks_0_attn2_to_v\n", + "lora_unet_mid_block_attentions_0_transformer_blocks_0_attn2_to_out_0\n", + "lora_unet_mid_block_attentions_0_transformer_blocks_0_ff_net_0_proj\n", + "lora_unet_mid_block_attentions_0_transformer_blocks_0_ff_net_2\n", + "lora_unet_mid_block_attentions_0_proj_out\n", + "Create SPM for U-Net: 192 modules.\n", + "use 8-bit AdamW optimizer | {}\n", + "Prompts\n", + "target='teen' positive='teen' unconditional='' neutral='adult' action='erase_with_la' guidance_scale=1.5 resolution=512 dynamic_resolution=True batch_size=4 dynamic_crops=False use_template=False la_strength=250.0 sampling_batch_size=4 seed=None case_number=0\n", + "target='child' positive='child' unconditional='' neutral='adult' action='erase_with_la' guidance_scale=1.5 resolution=512 dynamic_resolution=True batch_size=4 dynamic_crops=False use_template=False la_strength=250.0 sampling_batch_size=4 seed=None case_number=0\n", + "target='kid' positive='kid' unconditional='' neutral='adult' action='erase_with_la' guidance_scale=1.5 resolution=512 dynamic_resolution=True batch_size=4 dynamic_crops=False use_template=False la_strength=250.0 sampling_batch_size=4 seed=None case_number=0\n", + "target='loli' positive='loli' unconditional='' neutral='adult woman' action='erase_with_la' guidance_scale=1.5 resolution=512 dynamic_resolution=True batch_size=4 dynamic_crops=False use_template=False la_strength=250.0 sampling_batch_size=4 seed=None case_number=0\n", + "target='toddler' positive='toddler' unconditional='' neutral='adult' action='erase_with_la' guidance_scale=1.5 resolution=512 dynamic_resolution=True batch_size=4 dynamic_crops=False use_template=False la_strength=250.0 sampling_batch_size=4 seed=None case_number=0\n", + "target='preteen' positive='preteen' unconditional='' neutral='adult' action='erase_with_la' guidance_scale=1.5 resolution=512 dynamic_resolution=True batch_size=4 dynamic_crops=False use_template=False la_strength=250.0 sampling_batch_size=4 seed=None case_number=0\n", + "target='baby' positive='baby' unconditional='' neutral='adult' action='erase_with_la' guidance_scale=1.5 resolution=512 dynamic_resolution=True batch_size=4 dynamic_crops=False use_template=False la_strength=250.0 sampling_batch_size=4 seed=None case_number=0\n", + "target='newborn' positive='newborn' unconditional='' neutral='adult' action='erase_with_la' guidance_scale=1.5 resolution=512 dynamic_resolution=True batch_size=4 dynamic_crops=False use_template=False la_strength=250.0 sampling_batch_size=4 seed=None case_number=0\n", + "target='girl' positive='girl' unconditional='' neutral='woman' action='erase_with_la' guidance_scale=1.5 resolution=512 dynamic_resolution=True batch_size=4 dynamic_crops=False use_template=False la_strength=250.0 sampling_batch_size=4 seed=None case_number=0\n", + "target='boy' positive='boy' unconditional='' neutral='man' action='erase_with_la' guidance_scale=1.5 resolution=512 dynamic_resolution=True batch_size=4 dynamic_crops=False use_template=False la_strength=250.0 sampling_batch_size=4 seed=None case_number=0\n", + "target='young girl' positive='young girl' unconditional='' neutral='woman' action='erase_with_la' guidance_scale=1.5 resolution=512 dynamic_resolution=True batch_size=4 dynamic_crops=False use_template=False la_strength=250.0 sampling_batch_size=4 seed=None case_number=0\n", + "target='young boy' positive='young boy' unconditional='' neutral='man' action='erase_with_la' guidance_scale=1.5 resolution=512 dynamic_resolution=True batch_size=4 dynamic_crops=False use_template=False la_strength=250.0 sampling_batch_size=4 seed=None case_number=0\n", + "target='year old' positive='year old' unconditional='' neutral='20 year old' action='erase_with_la' guidance_scale=1.5 resolution=512 dynamic_resolution=True batch_size=4 dynamic_crops=False use_template=False la_strength=250.0 sampling_batch_size=4 seed=None case_number=0\n", + "target='years old' positive='years old' unconditional='' neutral='20 years old' action='erase_with_la' guidance_scale=1.5 resolution=512 dynamic_resolution=True batch_size=4 dynamic_crops=False use_template=False la_strength=250.0 sampling_batch_size=4 seed=None case_number=0\n", + "target='little boy' positive='little boy' unconditional='' neutral='man' action='erase_with_la' guidance_scale=1.5 resolution=512 dynamic_resolution=True batch_size=4 dynamic_crops=False use_template=False la_strength=250.0 sampling_batch_size=4 seed=None case_number=0\n", + "target='little girl' positive='little girl' unconditional='' neutral='woman' action='erase_with_la' guidance_scale=1.5 resolution=512 dynamic_resolution=True batch_size=4 dynamic_crops=False use_template=False la_strength=250.0 sampling_batch_size=4 seed=None case_number=0\n", + "norm of target: 249.0\n", + "norm of target: 249.0\n", + "norm of target: 250.0\n", + "norm of target: 250.0\n", + "norm of target: 251.0\n", + "norm of target: 248.0\n", + "norm of target: 250.0\n", + "norm of target: 249.0\n", + "norm of target: 248.0\n", + "norm of target: 250.0\n", + "norm of target: 248.0\n", + "norm of target: 250.0\n", + "norm of target: 247.0\n", + "norm of target: 249.0\n", + "norm of target: 251.0\n", + "norm of target: 249.0\n", + " 0%| | 0/6000 [00:00 Date: Tue, 11 Mar 2025 11:39:16 +0100 Subject: [PATCH 03/14] Cleared Cell Output and renamed config --- sd15_child_removal.ipynb | 33861 +------------------------------------ 1 file changed, 6 insertions(+), 33855 deletions(-) diff --git a/sd15_child_removal.ipynb b/sd15_child_removal.ipynb index ae4bf26..f3edbcc 100644 --- a/sd15_child_removal.ipynb +++ b/sd15_child_removal.ipynb @@ -49,7 +49,7 @@ "metadata": {}, "outputs": [], "source": [ - "pretrained_sd_model = r\"/media/alex/d5f7850c-8e1d-4f90-acd3-58912cc12a4a/ComfyUI/models/checkpoints/URPMv1.5.4.safetensors\" #@param {type: \"string\"}\n", + "pretrained_sd_model = r\"/path/to/model/URPMv1.5.4.safetensors\" #@param {type: \"string\"}\n", "is_v2_model = \"false\" #@param [\"true\", \"false\"]\n", "is_v_prediction_model = \"false\" #@param [\"true\", \"false\"]" ] @@ -179,31 +179,15 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": { "cellView": "form", "id": "a1pK61X_VY2L" }, - "outputs": [ - { - "ename": "NameError", - "evalue": "name 'generate_num' is not defined", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[7], line 51\u001b[0m\n\u001b[1;32m 10\u001b[0m config_file_path \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mconfigs/\u001b[39m\u001b[38;5;132;01m{\u001b[39;00msave_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m/config.yaml\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 11\u001b[0m prompts_file_path \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mconfigs/\u001b[39m\u001b[38;5;132;01m{\u001b[39;00msave_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m/prompt.yaml\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 14\u001b[0m config_file_content \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\"\"\u001b[39m\n\u001b[1;32m 15\u001b[0m \u001b[38;5;124mprompts_file: \u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mprompts_file_path\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 16\u001b[0m \n\u001b[1;32m 17\u001b[0m \u001b[38;5;124mpretrained_model:\u001b[39m\n\u001b[1;32m 18\u001b[0m \u001b[38;5;124m name_or_path: \u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mpretrained_sd_model\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 19\u001b[0m \u001b[38;5;124m v2: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mis_v2_model\u001b[38;5;132;01m}\u001b[39;00m\n\u001b[1;32m 20\u001b[0m \u001b[38;5;124m v_pred: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mis_v_prediction_model\u001b[38;5;132;01m}\u001b[39;00m\n\u001b[1;32m 21\u001b[0m \u001b[38;5;124m clip_skip: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mclip_skip\u001b[38;5;132;01m}\u001b[39;00m\n\u001b[1;32m 22\u001b[0m \n\u001b[1;32m 23\u001b[0m \u001b[38;5;124mnetwork:\u001b[39m\n\u001b[1;32m 24\u001b[0m \u001b[38;5;124m rank: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mdim\u001b[38;5;132;01m}\u001b[39;00m\n\u001b[1;32m 25\u001b[0m \u001b[38;5;124m alpha: 1.0\u001b[39m\n\u001b[1;32m 26\u001b[0m \n\u001b[1;32m 27\u001b[0m \u001b[38;5;124mtrain:\u001b[39m\n\u001b[1;32m 28\u001b[0m \u001b[38;5;124m precision: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mprecision\u001b[38;5;132;01m}\u001b[39;00m\n\u001b[1;32m 29\u001b[0m \u001b[38;5;124m noise_scheduler: \u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mddim\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 30\u001b[0m \u001b[38;5;124m iterations: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00miterations\u001b[38;5;132;01m}\u001b[39;00m\n\u001b[1;32m 31\u001b[0m \u001b[38;5;124m batch_size: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mbatch_size\u001b[38;5;132;01m}\u001b[39;00m\n\u001b[1;32m 32\u001b[0m \u001b[38;5;124m lr: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mlr\u001b[38;5;132;01m}\u001b[39;00m\n\u001b[1;32m 33\u001b[0m \u001b[38;5;124m unet_lr: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mlr\u001b[38;5;132;01m}\u001b[39;00m\n\u001b[1;32m 34\u001b[0m \u001b[38;5;124m text_encoder_lr: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;241m0.5\u001b[39m\u001b[38;5;250m \u001b[39m\u001b[38;5;241m*\u001b[39m\u001b[38;5;250m \u001b[39mlr\u001b[38;5;132;01m}\u001b[39;00m\n\u001b[1;32m 35\u001b[0m \u001b[38;5;124m optimizer_type: \u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00moptimizer\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 36\u001b[0m \u001b[38;5;124m lr_scheduler: \u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mlr_scheduler\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 37\u001b[0m \u001b[38;5;124m lr_warmup_steps: 500\u001b[39m\n\u001b[1;32m 38\u001b[0m \u001b[38;5;124m lr_scheduler_num_cycles: 3\u001b[39m\n\u001b[1;32m 39\u001b[0m \u001b[38;5;124m max_denoising_steps: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmax_denoising_steps\u001b[38;5;132;01m}\u001b[39;00m\n\u001b[1;32m 40\u001b[0m \n\u001b[1;32m 41\u001b[0m \u001b[38;5;124msave:\u001b[39m\n\u001b[1;32m 42\u001b[0m \u001b[38;5;124m name: \u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00msave_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 43\u001b[0m \u001b[38;5;124m path: \u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutput/\u001b[39m\u001b[38;5;132;01m{\u001b[39;00msave_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 44\u001b[0m \u001b[38;5;124m per_steps: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00msave_per_steps\u001b[38;5;132;01m}\u001b[39;00m\n\u001b[1;32m 45\u001b[0m \u001b[38;5;124m precision: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mprecision\u001b[38;5;132;01m}\u001b[39;00m\n\u001b[1;32m 46\u001b[0m \n\u001b[1;32m 47\u001b[0m \u001b[38;5;124mlogging:\u001b[39m\n\u001b[1;32m 48\u001b[0m \u001b[38;5;124m use_wandb: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfalse\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\n\u001b[1;32m 49\u001b[0m \u001b[38;5;124m interval: 500\u001b[39m\n\u001b[1;32m 50\u001b[0m \u001b[38;5;124m seed: 0\u001b[39m\n\u001b[0;32m---> 51\u001b[0m \u001b[38;5;124m generate_num: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[43mgenerate_num\u001b[49m\u001b[38;5;132;01m}\u001b[39;00m\n\u001b[1;32m 52\u001b[0m \u001b[38;5;124m run_name: \u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mrun_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 53\u001b[0m \u001b[38;5;124m verbose: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mverbose\u001b[38;5;132;01m}\u001b[39;00m\n\u001b[1;32m 54\u001b[0m \u001b[38;5;124m prompts: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mprompts_to_visualize\u001b[38;5;132;01m}\u001b[39;00m\n\u001b[1;32m 55\u001b[0m \n\u001b[1;32m 56\u001b[0m \u001b[38;5;124mother:\u001b[39m\n\u001b[1;32m 57\u001b[0m \u001b[38;5;124m use_xformers: true\u001b[39m\n\u001b[1;32m 58\u001b[0m \u001b[38;5;124m\"\"\"\u001b[39m\n\u001b[1;32m 60\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mos\u001b[39;00m\n\u001b[1;32m 61\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mexists(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m./configs/\u001b[39m\u001b[38;5;132;01m{\u001b[39;00msave_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m):\n", - "\u001b[0;31mNameError\u001b[0m: name 'generate_num' is not defined" - ] - } - ], + "outputs": [], "source": [ "# you can custom these strings to distiguish your different exps\n", - "exp_name = target_concept.replace(\" \", \"_\") if not multiple_concepts else \"multi_concepts\"\n", - "if multiple_concepts:\n", - " exp_name += \"_to_multiple_concepts\"\n", - "elif surrogate_concept:\n", - " exp_name += f\"_to_{surrogate_concept.replace(' ', '_')}\"\n", + "exp_name = f\"{sd15_child_removal_v3}\"\n", "save_name = f\"{exp_name}\"\n", "run_name = f\"{exp_name}\"\n", "\n", @@ -312,33844 +296,11 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": { "id": "WchSOvSHVY2L" }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "/home/alex/SPM/./train_spm.py:41: PydanticDeprecatedSince20: The `json` method is deprecated; use `model_dump_json` instead. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.10/migration/\n", - " \"prompts\": \",\".join([prompt.json() for prompt in prompts]),\n", - "/home/alex/SPM/./train_spm.py:42: PydanticDeprecatedSince20: The `json` method is deprecated; use `model_dump_json` instead. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.10/migration/\n", - " \"config\": config.json(),\n", - "Loading local checkpoint from /media/alex/d5f7850c-8e1d-4f90-acd3-58912cc12a4a/ComfyUI/models/checkpoints/URPMv1.5.4.safetensors\n", - "Fetching 11 files: 100%|████████████████████| 11/11 [00:00<00:00, 279620.27it/s]\n", - "Some weights of the model checkpoint were not used when initializing CLIPTextModel: \n", - " ['text_model.embeddings.position_ids']\n", - "In this conversion only the non-EMA weights are extracted. If you want to instead extract the EMA weights (usually better for inference), please make sure to add the `--extract_ema` flag.\n", - "Loading pipeline components...: 100%|█████████████| 6/6 [00:00<00:00, 47.50it/s]\n", - "You have disabled the safety checker for by passing `safety_checker=None`. Ensure that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered results in services or applications open to the public. Both the diffusers team and Hugging Face strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling it only for use-cases that involve analyzing network behavior or auditing its results. For more information, please have a look at https://github.com/huggingface/diffusers/pull/254 .\n", - "lora_unet_down_blocks_0_attentions_0_proj_in\n", - "lora_unet_down_blocks_0_attentions_0_transformer_blocks_0_attn1_to_q\n", - "lora_unet_down_blocks_0_attentions_0_transformer_blocks_0_attn1_to_k\n", - "lora_unet_down_blocks_0_attentions_0_transformer_blocks_0_attn1_to_v\n", - "lora_unet_down_blocks_0_attentions_0_transformer_blocks_0_attn1_to_out_0\n", - "lora_unet_down_blocks_0_attentions_0_transformer_blocks_0_attn2_to_q\n", - "lora_unet_down_blocks_0_attentions_0_transformer_blocks_0_attn2_to_k\n", - "lora_unet_down_blocks_0_attentions_0_transformer_blocks_0_attn2_to_v\n", - "lora_unet_down_blocks_0_attentions_0_transformer_blocks_0_attn2_to_out_0\n", - "lora_unet_down_blocks_0_attentions_0_transformer_blocks_0_ff_net_0_proj\n", - "lora_unet_down_blocks_0_attentions_0_transformer_blocks_0_ff_net_2\n", - "lora_unet_down_blocks_0_attentions_0_proj_out\n", - "lora_unet_down_blocks_0_attentions_1_proj_in\n", - "lora_unet_down_blocks_0_attentions_1_transformer_blocks_0_attn1_to_q\n", - "lora_unet_down_blocks_0_attentions_1_transformer_blocks_0_attn1_to_k\n", - "lora_unet_down_blocks_0_attentions_1_transformer_blocks_0_attn1_to_v\n", - "lora_unet_down_blocks_0_attentions_1_transformer_blocks_0_attn1_to_out_0\n", - "lora_unet_down_blocks_0_attentions_1_transformer_blocks_0_attn2_to_q\n", - "lora_unet_down_blocks_0_attentions_1_transformer_blocks_0_attn2_to_k\n", - "lora_unet_down_blocks_0_attentions_1_transformer_blocks_0_attn2_to_v\n", - "lora_unet_down_blocks_0_attentions_1_transformer_blocks_0_attn2_to_out_0\n", - "lora_unet_down_blocks_0_attentions_1_transformer_blocks_0_ff_net_0_proj\n", - "lora_unet_down_blocks_0_attentions_1_transformer_blocks_0_ff_net_2\n", - "lora_unet_down_blocks_0_attentions_1_proj_out\n", - "lora_unet_down_blocks_1_attentions_0_proj_in\n", - "lora_unet_down_blocks_1_attentions_0_transformer_blocks_0_attn1_to_q\n", - "lora_unet_down_blocks_1_attentions_0_transformer_blocks_0_attn1_to_k\n", - "lora_unet_down_blocks_1_attentions_0_transformer_blocks_0_attn1_to_v\n", - "lora_unet_down_blocks_1_attentions_0_transformer_blocks_0_attn1_to_out_0\n", - "lora_unet_down_blocks_1_attentions_0_transformer_blocks_0_attn2_to_q\n", - "lora_unet_down_blocks_1_attentions_0_transformer_blocks_0_attn2_to_k\n", - "lora_unet_down_blocks_1_attentions_0_transformer_blocks_0_attn2_to_v\n", - "lora_unet_down_blocks_1_attentions_0_transformer_blocks_0_attn2_to_out_0\n", - "lora_unet_down_blocks_1_attentions_0_transformer_blocks_0_ff_net_0_proj\n", - "lora_unet_down_blocks_1_attentions_0_transformer_blocks_0_ff_net_2\n", - "lora_unet_down_blocks_1_attentions_0_proj_out\n", - "lora_unet_down_blocks_1_attentions_1_proj_in\n", - "lora_unet_down_blocks_1_attentions_1_transformer_blocks_0_attn1_to_q\n", - "lora_unet_down_blocks_1_attentions_1_transformer_blocks_0_attn1_to_k\n", - "lora_unet_down_blocks_1_attentions_1_transformer_blocks_0_attn1_to_v\n", - "lora_unet_down_blocks_1_attentions_1_transformer_blocks_0_attn1_to_out_0\n", - "lora_unet_down_blocks_1_attentions_1_transformer_blocks_0_attn2_to_q\n", - "lora_unet_down_blocks_1_attentions_1_transformer_blocks_0_attn2_to_k\n", - "lora_unet_down_blocks_1_attentions_1_transformer_blocks_0_attn2_to_v\n", - "lora_unet_down_blocks_1_attentions_1_transformer_blocks_0_attn2_to_out_0\n", - "lora_unet_down_blocks_1_attentions_1_transformer_blocks_0_ff_net_0_proj\n", - "lora_unet_down_blocks_1_attentions_1_transformer_blocks_0_ff_net_2\n", - "lora_unet_down_blocks_1_attentions_1_proj_out\n", - "lora_unet_down_blocks_2_attentions_0_proj_in\n", - "lora_unet_down_blocks_2_attentions_0_transformer_blocks_0_attn1_to_q\n", - "lora_unet_down_blocks_2_attentions_0_transformer_blocks_0_attn1_to_k\n", - "lora_unet_down_blocks_2_attentions_0_transformer_blocks_0_attn1_to_v\n", - "lora_unet_down_blocks_2_attentions_0_transformer_blocks_0_attn1_to_out_0\n", - "lora_unet_down_blocks_2_attentions_0_transformer_blocks_0_attn2_to_q\n", - "lora_unet_down_blocks_2_attentions_0_transformer_blocks_0_attn2_to_k\n", - "lora_unet_down_blocks_2_attentions_0_transformer_blocks_0_attn2_to_v\n", - "lora_unet_down_blocks_2_attentions_0_transformer_blocks_0_attn2_to_out_0\n", - "lora_unet_down_blocks_2_attentions_0_transformer_blocks_0_ff_net_0_proj\n", - "lora_unet_down_blocks_2_attentions_0_transformer_blocks_0_ff_net_2\n", - "lora_unet_down_blocks_2_attentions_0_proj_out\n", - "lora_unet_down_blocks_2_attentions_1_proj_in\n", - "lora_unet_down_blocks_2_attentions_1_transformer_blocks_0_attn1_to_q\n", - "lora_unet_down_blocks_2_attentions_1_transformer_blocks_0_attn1_to_k\n", - "lora_unet_down_blocks_2_attentions_1_transformer_blocks_0_attn1_to_v\n", - "lora_unet_down_blocks_2_attentions_1_transformer_blocks_0_attn1_to_out_0\n", - "lora_unet_down_blocks_2_attentions_1_transformer_blocks_0_attn2_to_q\n", - "lora_unet_down_blocks_2_attentions_1_transformer_blocks_0_attn2_to_k\n", - "lora_unet_down_blocks_2_attentions_1_transformer_blocks_0_attn2_to_v\n", - "lora_unet_down_blocks_2_attentions_1_transformer_blocks_0_attn2_to_out_0\n", - "lora_unet_down_blocks_2_attentions_1_transformer_blocks_0_ff_net_0_proj\n", - "lora_unet_down_blocks_2_attentions_1_transformer_blocks_0_ff_net_2\n", - "lora_unet_down_blocks_2_attentions_1_proj_out\n", - "lora_unet_up_blocks_1_attentions_0_proj_in\n", - "lora_unet_up_blocks_1_attentions_0_transformer_blocks_0_attn1_to_q\n", - "lora_unet_up_blocks_1_attentions_0_transformer_blocks_0_attn1_to_k\n", - "lora_unet_up_blocks_1_attentions_0_transformer_blocks_0_attn1_to_v\n", - "lora_unet_up_blocks_1_attentions_0_transformer_blocks_0_attn1_to_out_0\n", - "lora_unet_up_blocks_1_attentions_0_transformer_blocks_0_attn2_to_q\n", - "lora_unet_up_blocks_1_attentions_0_transformer_blocks_0_attn2_to_k\n", - "lora_unet_up_blocks_1_attentions_0_transformer_blocks_0_attn2_to_v\n", - "lora_unet_up_blocks_1_attentions_0_transformer_blocks_0_attn2_to_out_0\n", - "lora_unet_up_blocks_1_attentions_0_transformer_blocks_0_ff_net_0_proj\n", - "lora_unet_up_blocks_1_attentions_0_transformer_blocks_0_ff_net_2\n", - "lora_unet_up_blocks_1_attentions_0_proj_out\n", - "lora_unet_up_blocks_1_attentions_1_proj_in\n", - "lora_unet_up_blocks_1_attentions_1_transformer_blocks_0_attn1_to_q\n", - "lora_unet_up_blocks_1_attentions_1_transformer_blocks_0_attn1_to_k\n", - "lora_unet_up_blocks_1_attentions_1_transformer_blocks_0_attn1_to_v\n", - "lora_unet_up_blocks_1_attentions_1_transformer_blocks_0_attn1_to_out_0\n", - "lora_unet_up_blocks_1_attentions_1_transformer_blocks_0_attn2_to_q\n", - "lora_unet_up_blocks_1_attentions_1_transformer_blocks_0_attn2_to_k\n", - "lora_unet_up_blocks_1_attentions_1_transformer_blocks_0_attn2_to_v\n", - "lora_unet_up_blocks_1_attentions_1_transformer_blocks_0_attn2_to_out_0\n", - "lora_unet_up_blocks_1_attentions_1_transformer_blocks_0_ff_net_0_proj\n", - "lora_unet_up_blocks_1_attentions_1_transformer_blocks_0_ff_net_2\n", - "lora_unet_up_blocks_1_attentions_1_proj_out\n", - "lora_unet_up_blocks_1_attentions_2_proj_in\n", - "lora_unet_up_blocks_1_attentions_2_transformer_blocks_0_attn1_to_q\n", - "lora_unet_up_blocks_1_attentions_2_transformer_blocks_0_attn1_to_k\n", - "lora_unet_up_blocks_1_attentions_2_transformer_blocks_0_attn1_to_v\n", - "lora_unet_up_blocks_1_attentions_2_transformer_blocks_0_attn1_to_out_0\n", - "lora_unet_up_blocks_1_attentions_2_transformer_blocks_0_attn2_to_q\n", - "lora_unet_up_blocks_1_attentions_2_transformer_blocks_0_attn2_to_k\n", - "lora_unet_up_blocks_1_attentions_2_transformer_blocks_0_attn2_to_v\n", - "lora_unet_up_blocks_1_attentions_2_transformer_blocks_0_attn2_to_out_0\n", - "lora_unet_up_blocks_1_attentions_2_transformer_blocks_0_ff_net_0_proj\n", - "lora_unet_up_blocks_1_attentions_2_transformer_blocks_0_ff_net_2\n", - "lora_unet_up_blocks_1_attentions_2_proj_out\n", - "lora_unet_up_blocks_2_attentions_0_proj_in\n", - "lora_unet_up_blocks_2_attentions_0_transformer_blocks_0_attn1_to_q\n", - "lora_unet_up_blocks_2_attentions_0_transformer_blocks_0_attn1_to_k\n", - "lora_unet_up_blocks_2_attentions_0_transformer_blocks_0_attn1_to_v\n", - "lora_unet_up_blocks_2_attentions_0_transformer_blocks_0_attn1_to_out_0\n", - "lora_unet_up_blocks_2_attentions_0_transformer_blocks_0_attn2_to_q\n", - "lora_unet_up_blocks_2_attentions_0_transformer_blocks_0_attn2_to_k\n", - "lora_unet_up_blocks_2_attentions_0_transformer_blocks_0_attn2_to_v\n", - "lora_unet_up_blocks_2_attentions_0_transformer_blocks_0_attn2_to_out_0\n", - "lora_unet_up_blocks_2_attentions_0_transformer_blocks_0_ff_net_0_proj\n", - "lora_unet_up_blocks_2_attentions_0_transformer_blocks_0_ff_net_2\n", - "lora_unet_up_blocks_2_attentions_0_proj_out\n", - "lora_unet_up_blocks_2_attentions_1_proj_in\n", - "lora_unet_up_blocks_2_attentions_1_transformer_blocks_0_attn1_to_q\n", - "lora_unet_up_blocks_2_attentions_1_transformer_blocks_0_attn1_to_k\n", - "lora_unet_up_blocks_2_attentions_1_transformer_blocks_0_attn1_to_v\n", - "lora_unet_up_blocks_2_attentions_1_transformer_blocks_0_attn1_to_out_0\n", - "lora_unet_up_blocks_2_attentions_1_transformer_blocks_0_attn2_to_q\n", - "lora_unet_up_blocks_2_attentions_1_transformer_blocks_0_attn2_to_k\n", - "lora_unet_up_blocks_2_attentions_1_transformer_blocks_0_attn2_to_v\n", - "lora_unet_up_blocks_2_attentions_1_transformer_blocks_0_attn2_to_out_0\n", - "lora_unet_up_blocks_2_attentions_1_transformer_blocks_0_ff_net_0_proj\n", - "lora_unet_up_blocks_2_attentions_1_transformer_blocks_0_ff_net_2\n", - "lora_unet_up_blocks_2_attentions_1_proj_out\n", - "lora_unet_up_blocks_2_attentions_2_proj_in\n", - "lora_unet_up_blocks_2_attentions_2_transformer_blocks_0_attn1_to_q\n", - "lora_unet_up_blocks_2_attentions_2_transformer_blocks_0_attn1_to_k\n", - "lora_unet_up_blocks_2_attentions_2_transformer_blocks_0_attn1_to_v\n", - "lora_unet_up_blocks_2_attentions_2_transformer_blocks_0_attn1_to_out_0\n", - "lora_unet_up_blocks_2_attentions_2_transformer_blocks_0_attn2_to_q\n", - "lora_unet_up_blocks_2_attentions_2_transformer_blocks_0_attn2_to_k\n", - "lora_unet_up_blocks_2_attentions_2_transformer_blocks_0_attn2_to_v\n", - "lora_unet_up_blocks_2_attentions_2_transformer_blocks_0_attn2_to_out_0\n", - "lora_unet_up_blocks_2_attentions_2_transformer_blocks_0_ff_net_0_proj\n", - "lora_unet_up_blocks_2_attentions_2_transformer_blocks_0_ff_net_2\n", - "lora_unet_up_blocks_2_attentions_2_proj_out\n", - "lora_unet_up_blocks_3_attentions_0_proj_in\n", - "lora_unet_up_blocks_3_attentions_0_transformer_blocks_0_attn1_to_q\n", - "lora_unet_up_blocks_3_attentions_0_transformer_blocks_0_attn1_to_k\n", - "lora_unet_up_blocks_3_attentions_0_transformer_blocks_0_attn1_to_v\n", - "lora_unet_up_blocks_3_attentions_0_transformer_blocks_0_attn1_to_out_0\n", - "lora_unet_up_blocks_3_attentions_0_transformer_blocks_0_attn2_to_q\n", - "lora_unet_up_blocks_3_attentions_0_transformer_blocks_0_attn2_to_k\n", - "lora_unet_up_blocks_3_attentions_0_transformer_blocks_0_attn2_to_v\n", - "lora_unet_up_blocks_3_attentions_0_transformer_blocks_0_attn2_to_out_0\n", - "lora_unet_up_blocks_3_attentions_0_transformer_blocks_0_ff_net_0_proj\n", - "lora_unet_up_blocks_3_attentions_0_transformer_blocks_0_ff_net_2\n", - "lora_unet_up_blocks_3_attentions_0_proj_out\n", - "lora_unet_up_blocks_3_attentions_1_proj_in\n", - "lora_unet_up_blocks_3_attentions_1_transformer_blocks_0_attn1_to_q\n", - "lora_unet_up_blocks_3_attentions_1_transformer_blocks_0_attn1_to_k\n", - "lora_unet_up_blocks_3_attentions_1_transformer_blocks_0_attn1_to_v\n", - "lora_unet_up_blocks_3_attentions_1_transformer_blocks_0_attn1_to_out_0\n", - "lora_unet_up_blocks_3_attentions_1_transformer_blocks_0_attn2_to_q\n", - "lora_unet_up_blocks_3_attentions_1_transformer_blocks_0_attn2_to_k\n", - "lora_unet_up_blocks_3_attentions_1_transformer_blocks_0_attn2_to_v\n", - "lora_unet_up_blocks_3_attentions_1_transformer_blocks_0_attn2_to_out_0\n", - "lora_unet_up_blocks_3_attentions_1_transformer_blocks_0_ff_net_0_proj\n", - "lora_unet_up_blocks_3_attentions_1_transformer_blocks_0_ff_net_2\n", - "lora_unet_up_blocks_3_attentions_1_proj_out\n", - "lora_unet_up_blocks_3_attentions_2_proj_in\n", - "lora_unet_up_blocks_3_attentions_2_transformer_blocks_0_attn1_to_q\n", - "lora_unet_up_blocks_3_attentions_2_transformer_blocks_0_attn1_to_k\n", - "lora_unet_up_blocks_3_attentions_2_transformer_blocks_0_attn1_to_v\n", - "lora_unet_up_blocks_3_attentions_2_transformer_blocks_0_attn1_to_out_0\n", - "lora_unet_up_blocks_3_attentions_2_transformer_blocks_0_attn2_to_q\n", - "lora_unet_up_blocks_3_attentions_2_transformer_blocks_0_attn2_to_k\n", - "lora_unet_up_blocks_3_attentions_2_transformer_blocks_0_attn2_to_v\n", - "lora_unet_up_blocks_3_attentions_2_transformer_blocks_0_attn2_to_out_0\n", - "lora_unet_up_blocks_3_attentions_2_transformer_blocks_0_ff_net_0_proj\n", - "lora_unet_up_blocks_3_attentions_2_transformer_blocks_0_ff_net_2\n", - "lora_unet_up_blocks_3_attentions_2_proj_out\n", - "lora_unet_mid_block_attentions_0_proj_in\n", - "lora_unet_mid_block_attentions_0_transformer_blocks_0_attn1_to_q\n", - "lora_unet_mid_block_attentions_0_transformer_blocks_0_attn1_to_k\n", - "lora_unet_mid_block_attentions_0_transformer_blocks_0_attn1_to_v\n", - "lora_unet_mid_block_attentions_0_transformer_blocks_0_attn1_to_out_0\n", - "lora_unet_mid_block_attentions_0_transformer_blocks_0_attn2_to_q\n", - "lora_unet_mid_block_attentions_0_transformer_blocks_0_attn2_to_k\n", - "lora_unet_mid_block_attentions_0_transformer_blocks_0_attn2_to_v\n", - "lora_unet_mid_block_attentions_0_transformer_blocks_0_attn2_to_out_0\n", - "lora_unet_mid_block_attentions_0_transformer_blocks_0_ff_net_0_proj\n", - "lora_unet_mid_block_attentions_0_transformer_blocks_0_ff_net_2\n", - "lora_unet_mid_block_attentions_0_proj_out\n", - "Create SPM for U-Net: 192 modules.\n", - "use 8-bit AdamW optimizer | {}\n", - "Prompts\n", - "target='teen' positive='teen' unconditional='' neutral='adult' action='erase_with_la' guidance_scale=1.5 resolution=512 dynamic_resolution=True batch_size=4 dynamic_crops=False use_template=False la_strength=250.0 sampling_batch_size=4 seed=None case_number=0\n", - "target='child' positive='child' unconditional='' neutral='adult' action='erase_with_la' guidance_scale=1.5 resolution=512 dynamic_resolution=True batch_size=4 dynamic_crops=False use_template=False la_strength=250.0 sampling_batch_size=4 seed=None case_number=0\n", - "target='kid' positive='kid' unconditional='' neutral='adult' action='erase_with_la' guidance_scale=1.5 resolution=512 dynamic_resolution=True batch_size=4 dynamic_crops=False use_template=False la_strength=250.0 sampling_batch_size=4 seed=None case_number=0\n", - "target='loli' positive='loli' unconditional='' neutral='adult woman' action='erase_with_la' guidance_scale=1.5 resolution=512 dynamic_resolution=True batch_size=4 dynamic_crops=False use_template=False la_strength=250.0 sampling_batch_size=4 seed=None case_number=0\n", - "target='toddler' positive='toddler' unconditional='' neutral='adult' action='erase_with_la' guidance_scale=1.5 resolution=512 dynamic_resolution=True batch_size=4 dynamic_crops=False use_template=False la_strength=250.0 sampling_batch_size=4 seed=None case_number=0\n", - "target='preteen' positive='preteen' unconditional='' neutral='adult' action='erase_with_la' guidance_scale=1.5 resolution=512 dynamic_resolution=True batch_size=4 dynamic_crops=False use_template=False la_strength=250.0 sampling_batch_size=4 seed=None case_number=0\n", - "target='baby' positive='baby' unconditional='' neutral='adult' action='erase_with_la' guidance_scale=1.5 resolution=512 dynamic_resolution=True batch_size=4 dynamic_crops=False use_template=False la_strength=250.0 sampling_batch_size=4 seed=None case_number=0\n", - "target='newborn' positive='newborn' unconditional='' neutral='adult' action='erase_with_la' guidance_scale=1.5 resolution=512 dynamic_resolution=True batch_size=4 dynamic_crops=False use_template=False la_strength=250.0 sampling_batch_size=4 seed=None case_number=0\n", - "target='girl' positive='girl' unconditional='' neutral='woman' action='erase_with_la' guidance_scale=1.5 resolution=512 dynamic_resolution=True batch_size=4 dynamic_crops=False use_template=False la_strength=250.0 sampling_batch_size=4 seed=None case_number=0\n", - "target='boy' positive='boy' unconditional='' neutral='man' action='erase_with_la' guidance_scale=1.5 resolution=512 dynamic_resolution=True batch_size=4 dynamic_crops=False use_template=False la_strength=250.0 sampling_batch_size=4 seed=None case_number=0\n", - "target='young girl' positive='young girl' unconditional='' neutral='woman' action='erase_with_la' guidance_scale=1.5 resolution=512 dynamic_resolution=True batch_size=4 dynamic_crops=False use_template=False la_strength=250.0 sampling_batch_size=4 seed=None case_number=0\n", - "target='young boy' positive='young boy' unconditional='' neutral='man' action='erase_with_la' guidance_scale=1.5 resolution=512 dynamic_resolution=True batch_size=4 dynamic_crops=False use_template=False la_strength=250.0 sampling_batch_size=4 seed=None case_number=0\n", - "target='year old' positive='year old' unconditional='' neutral='20 year old' action='erase_with_la' guidance_scale=1.5 resolution=512 dynamic_resolution=True batch_size=4 dynamic_crops=False use_template=False la_strength=250.0 sampling_batch_size=4 seed=None case_number=0\n", - "target='years old' positive='years old' unconditional='' neutral='20 years old' action='erase_with_la' guidance_scale=1.5 resolution=512 dynamic_resolution=True batch_size=4 dynamic_crops=False use_template=False la_strength=250.0 sampling_batch_size=4 seed=None case_number=0\n", - "target='little boy' positive='little boy' unconditional='' neutral='man' action='erase_with_la' guidance_scale=1.5 resolution=512 dynamic_resolution=True batch_size=4 dynamic_crops=False use_template=False la_strength=250.0 sampling_batch_size=4 seed=None case_number=0\n", - "target='little girl' positive='little girl' unconditional='' neutral='woman' action='erase_with_la' guidance_scale=1.5 resolution=512 dynamic_resolution=True batch_size=4 dynamic_crops=False use_template=False la_strength=250.0 sampling_batch_size=4 seed=None case_number=0\n", - "norm of target: 249.0\n", - "norm of target: 249.0\n", - "norm of target: 250.0\n", - "norm of target: 250.0\n", - "norm of target: 251.0\n", - "norm of target: 248.0\n", - "norm of target: 250.0\n", - "norm of target: 249.0\n", - "norm of target: 248.0\n", - "norm of target: 250.0\n", - "norm of target: 248.0\n", - "norm of target: 250.0\n", - "norm of target: 247.0\n", - "norm of target: 249.0\n", - "norm of target: 251.0\n", - "norm of target: 249.0\n", - " 0%| | 0/6000 [00:00 Date: Tue, 11 Mar 2025 11:54:26 +0100 Subject: [PATCH 04/14] Fixed config name and cleared more cell outputs --- sd15_child_removal.ipynb | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/sd15_child_removal.ipynb b/sd15_child_removal.ipynb index f3edbcc..5cced97 100644 --- a/sd15_child_removal.ipynb +++ b/sd15_child_removal.ipynb @@ -45,7 +45,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -66,7 +66,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -95,7 +95,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -118,7 +118,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -151,21 +151,21 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "#wandb_token = \"\" #@param {type: \"string\"}\n", + "wandb_token = \"\" #@param {type: \"string\"}\n", "\n", - "#prompts_to_visualize = [\"cat\", \"dog\"] #@param {type: \"string\"}\n", - "#generate_num = 2 #@param {type: \"number\"}\n", + "prompts_to_visualize = [\"cat\", \"dog\"] #@param {type: \"string\"}\n", + "generate_num = 2 #@param {type: \"number\"}\n", "\n", "## track target & surrogate by default\n", - "#prompts_to_visualize = [target_concept, surrogate_concept] + prompts_to_visualize\n", + "prompts_to_visualize = [target_concept, surrogate_concept] + prompts_to_visualize\n", "\n", "## login with your wandb token\n", - "#if wandb_token != \"\": \n", - "# !wandb login {wandb_token}\n" + "if wandb_token != \"\": \n", + " !wandb login {wandb_token}\n" ] }, { @@ -187,7 +187,7 @@ "outputs": [], "source": [ "# you can custom these strings to distiguish your different exps\n", - "exp_name = f\"{sd15_child_removal_v3}\"\n", + "exp_name = f\"sd15_child_removal_v3\"\n", "save_name = f\"{exp_name}\"\n", "run_name = f\"{exp_name}\"\n", "\n", @@ -229,7 +229,7 @@ " precision: {precision}\n", "\n", "logging:\n", - " use_wandb: {\"false\"}\n", + " use_wandb: \"false\"\n", " interval: 500\n", " seed: 0\n", " generate_num: {generate_num}\n", From c98e5f785e2a6bcd812190a8ff359d09e2c32b24 Mon Sep 17 00:00:00 2001 From: saftle Date: Tue, 11 Mar 2025 13:31:34 +0100 Subject: [PATCH 05/14] Fixed a few cells in the notebook --- sd15_child_removal.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sd15_child_removal.ipynb b/sd15_child_removal.ipynb index 5cced97..c0cca18 100644 --- a/sd15_child_removal.ipynb +++ b/sd15_child_removal.ipynb @@ -49,7 +49,7 @@ "metadata": {}, "outputs": [], "source": [ - "pretrained_sd_model = r\"/path/to/model/URPMv1.5.4.safetensors\" #@param {type: \"string\"}\n", + "pretrained_sd_model = r\"/path/to/model/model.safetensors\" #@param {type: \"string\"}\n", "is_v2_model = \"false\" #@param [\"true\", \"false\"]\n", "is_v_prediction_model = \"false\" #@param [\"true\", \"false\"]" ] @@ -676,7 +676,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.16" + "version": "3.10.12" }, "widgets": { "application/vnd.jupyter.widget-state+json": { From cb79eba78639513efe18abf1b0474fa49c9a13ae Mon Sep 17 00:00:00 2001 From: saftle Date: Fri, 28 Mar 2025 20:35:54 +0100 Subject: [PATCH 06/14] Update requirements.txt --- requirements.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 139dbcf..eaf6eb4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,4 +28,6 @@ lightning nudenet==3.0.8 invisible-watermark git+https://github.com/openai/CLIP.git - +torchvision --index-url https://download.pytorch.org/whl/cu118" +xformers --index-url https://download.pytorch.org/whl/cu118" +huggingface_hub==0.25.0 From 71f35b2fb5ef1f67998f1a180a144f522962f279 Mon Sep 17 00:00:00 2001 From: saftle Date: Fri, 28 Mar 2025 20:37:30 +0100 Subject: [PATCH 07/14] Update requirements.txt --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index eaf6eb4..01b8b2c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,6 @@ diffusers==0.18.2 embedding_reader==1.5.1 fire==0.5.0 fsspec==2023.5.0 -library==0.0.0 lion_pytorch==0.0.6 matplotlib==3.7.1 multidict==6.0.4 From e1fdcd63e02ecdef6238b666a5c391eb8ff5ed72 Mon Sep 17 00:00:00 2001 From: saftle Date: Fri, 28 Mar 2025 20:37:55 +0100 Subject: [PATCH 08/14] Update setup.sh --- setup.sh | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/setup.sh b/setup.sh index 20b47c7..fb123c4 100644 --- a/setup.sh +++ b/setup.sh @@ -31,16 +31,5 @@ function run_in_conda() { git clone https://github.com/civitai/SPM cd SPM/ -# Install packages within the conda environment -run_in_conda "pip install -U pip" -run_in_conda "pip install torch --index-url https://download.pytorch.org/whl/cu118" -run_in_conda "pip install torchvision --index-url https://download.pytorch.org/whl/cu118" -run_in_conda "pip install xformers --index-url https://download.pytorch.org/whl/cu118" -run_in_conda "pip install huggingface_hub==0.25.0" - -# Modify requirements file to avoid conflicts -sed -i 's/^\(library==.*\)/#\1/' requirements.txt -sed 's/==[^# ]*//g' requirements.txt > requirements_nv.txt - # Install requirements within the conda environment -run_in_conda "pip install -r ./requirements_nv.txt" +run_in_conda "pip install -r ./requirements.txt" From 8e5165ce05a0d69511a88850518495df113ee0a1 Mon Sep 17 00:00:00 2001 From: saftle Date: Fri, 28 Mar 2025 20:41:00 +0100 Subject: [PATCH 09/14] Update requirements.txt --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 01b8b2c..ba830e9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,6 +27,6 @@ lightning nudenet==3.0.8 invisible-watermark git+https://github.com/openai/CLIP.git -torchvision --index-url https://download.pytorch.org/whl/cu118" -xformers --index-url https://download.pytorch.org/whl/cu118" +torchvision +xformers huggingface_hub==0.25.0 From 599c5bb912b09fd02f1cb6035f1cc3287ed1925b Mon Sep 17 00:00:00 2001 From: saftle Date: Fri, 28 Mar 2025 20:45:59 +0100 Subject: [PATCH 10/14] Update requirements.txt --- requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index ba830e9..ddb333d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,6 +27,5 @@ lightning nudenet==3.0.8 invisible-watermark git+https://github.com/openai/CLIP.git -torchvision -xformers huggingface_hub==0.25.0 +omegaconf From 82860c04ee81885ba2768267e0c8b1e97de0d058 Mon Sep 17 00:00:00 2001 From: saftle Date: Fri, 28 Mar 2025 20:53:50 +0100 Subject: [PATCH 11/14] Update requirements.txt --- requirements.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index ddb333d..7a1b373 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,5 +27,3 @@ lightning nudenet==3.0.8 invisible-watermark git+https://github.com/openai/CLIP.git -huggingface_hub==0.25.0 -omegaconf From a9240fec27f1d6df2c651397f6226b8fc2669304 Mon Sep 17 00:00:00 2001 From: saftle Date: Fri, 28 Mar 2025 21:00:55 +0100 Subject: [PATCH 12/14] Update requirements.txt --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 7a1b373..44f42d8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,3 +27,4 @@ lightning nudenet==3.0.8 invisible-watermark git+https://github.com/openai/CLIP.git +omegaconf From cae86aecbfd94a21f59a0a7fbcfa7f583bfcfc85 Mon Sep 17 00:00:00 2001 From: saftle Date: Fri, 28 Mar 2025 21:01:45 +0100 Subject: [PATCH 13/14] Update setup.sh --- setup.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/setup.sh b/setup.sh index fb123c4..be03538 100644 --- a/setup.sh +++ b/setup.sh @@ -27,9 +27,12 @@ function run_in_conda() { conda run -n spm --no-capture-output bash -c "$*" } -# Clone SPM repository -git clone https://github.com/civitai/SPM -cd SPM/ +# Install packages within the conda environment +run_in_conda "pip install -U pip" +run_in_conda "pip install torch --index-url https://download.pytorch.org/whl/cu118" +run_in_conda "pip install torchvision --index-url https://download.pytorch.org/whl/cu118" +run_in_conda "pip install xformers --index-url https://download.pytorch.org/whl/cu118" +run_in_conda "pip install huggingface_hub==0.25.0" # Install requirements within the conda environment run_in_conda "pip install -r ./requirements.txt" From 16a747d17415a649a5052213eaa3eafbc33e6fe1 Mon Sep 17 00:00:00 2001 From: saftle Date: Tue, 8 Apr 2025 12:59:32 +0200 Subject: [PATCH 14/14] Added SDXL Child Removal jupyter notebook --- sdxl_child_removal.ipynb | 31357 +++++++++++++++++++++++++++++++++++++ 1 file changed, 31357 insertions(+) create mode 100644 sdxl_child_removal.ipynb diff --git a/sdxl_child_removal.ipynb b/sdxl_child_removal.ipynb new file mode 100644 index 0000000..ab58bcd --- /dev/null +++ b/sdxl_child_removal.ipynb @@ -0,0 +1,31357 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "view-in-github" + }, + "source": [ + "\"Open" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "id": "_U-cYMtEVY2K" + }, + "source": [ + "## Create Configs" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "id": "XBgbejdDV2iq" + }, + "source": [ + "### Step 1: Choose Base Model\n", + "\n", + "Examples of `pretrained_sd_model`:\n", + "- SD v1.4: `CompVis/stable-diffusion-v1-4`\n", + "- SD v1.5: `runwayml/stable-diffusion-v1-5`\n", + "- Dreamshaper v8: `stablediffusionapi/dreamshaper-v8`\n", + "- SD v2.1: `stabilityai/stable-diffusion-2-1-base`\n", + "- WD1.5 beta3: `Birchlabs/wd-1-5-beta3-unofficial`\n", + "- SDXL: `stabilityai/stable-diffusion-xl-base-1.0`\n", + "- Juggernaut v6: `RunDiffusion/Juggernaut-XL-v6`\n", + "- PonyXL: `stablediffusionapi/pony-diffusion-v6-xl`\n", + "\n", + "If base model is v2.x, set `is_v2_model` to `true`." + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": {}, + "outputs": [], + "source": [ + "pretrained_sd_model = r\"/workspace/SPM/URPMXL-V6.safetensors\" #@param {type: \"string\"}\n", + "is_v2_model = \"false\" #@param [\"true\", \"false\"]\n", + "is_v_prediction_model = \"false\" #@param [\"true\", \"false\"]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Step 2: Choose Concept\n", + "\n", + "- `target_concept`: Targeted concept for erasing\n", + "- `surrogate_concept`: Surrogate concept for defining model generation after erasure, empty string by default" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "metadata": {}, + "outputs": [], + "source": [ + "# If you want to train multiple concepts at once in a single SPM, set this to True\n", + "multiple_concepts = True #@param {type: \"boolean\"}\n", + "\n", + "target_concept = '' #@param {type: \"string\"}\n", + "surrogate_concept = '' #@param {type: \"string\"}\n", + "\n", + "# If multiple_concepts is True, the following fields will be used instead\n", + "target_concepts = ['teen','child','kid','loli','toddler','preteen','baby','newborn','girl','boy','young girl','young boy','year old','years old','little boy','little girl'] #@param {type: \"string\"}\n", + "surrogate_concepts = ['adult','adult','adult','woman','adult','adult','adult','adult','woman','man','woman','man','20 year old','20 years old','man','woman'] #@param {type: \"string\"}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Step 3: SPM Settings\n", + "\n", + "- `mode`: `erase_with_la` or `erase` (for ablation)\n", + "- `dim`: default to 1, other values for ablation\n", + "- `sampling_batch_size`: indicates how many latent anchors are sampled for each iteration, default to 4, can be reduced if there's not enough VRAM\n", + "- `la_strength`: indicates the latent anchoring loss strength that balancing the erasure and preservation, default to 1000 for SD v1.4 models, should be further tuned for other base models for better performance\n" + ] + }, + { + "cell_type": "code", + "execution_count": 75, + "metadata": {}, + "outputs": [], + "source": [ + "mode = 'erase_with_la' #@param [\"erase_with_la\", \"erase\"]\n", + "dim = 2 #@param {type: \"number\"}\n", + "sampling_batch_size = 1 #@param {type: \"number\"}\n", + "la_strength = 400 #@param {type: \"number\"}\n", + "erasing_scale = 2.0 #@param {type: \"number\"}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Step 4: Training Settings\n", + "\n", + "There are two parts for training settings, SD settings and optimization settings.\n", + "Notice that `resolution` is set to 512 for SD v1.x, 768 for SD v2.x, and 1024 for SDXL." + ] + }, + { + "cell_type": "code", + "execution_count": 76, + "metadata": {}, + "outputs": [], + "source": [ + "resolution = 1024 #@param [512, 640, 768, 896, 960, 1024]\n", + "max_denoising_steps = 20 #@param {type: \"number\"}\n", + "dynamic_resolution = \"true\" #@param [\"true\", \"false\"]\n", + "clip_skip = 2 #@param [1, 2]\n", + "\n", + "batch_size = 1 #@param {type: \"number\"}\n", + "iterations = 5000 #@param {type: \"number\"}\n", + "lr = 15e-5 #@param {type: \"number\"}\n", + "optimizer = \"AdamW8bit\" #@param {type: \"string\"}\n", + "lr_scheduler = \"cosine_with_restarts\" #@param {type: \"string\"}\n", + "lr_warmup_steps = 300 #@param {type: \"number\"}\n", + "lr_scheduler_num_cycles = 3 #@param {type: \"number\"}\n", + "save_per_steps = 750 #@param {type: \"number\"}\n", + "precision = \"bfloat16\" #@param [\"float32\", \"float16\", \"bfloat16\"]\n", + "verbose = \"false\" #@param [\"true\", \"false\"]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Step 5: (Optional) Tracking Training Details with WandB\n", + "\n", + "You can setup your wandb token to track the training details, including training statistics (e.g. losses, learning rates) and visualizations.\n", + "Your wandb token can be retrieved from https://wandb.ai/authorize ." + ] + }, + { + "cell_type": "code", + "execution_count": 77, + "metadata": {}, + "outputs": [], + "source": [ + "wandb_token = \"\" #@param {type: \"string\"}\n", + "\n", + "prompts_to_visualize = [\"cat\", \"dog\"] #@param {type: \"string\"}\n", + "generate_num = 2 #@param {type: \"number\"}\n", + "\n", + "## track target & surrogate by default\n", + "prompts_to_visualize = [target_concept, surrogate_concept] + prompts_to_visualize\n", + "\n", + "## login with your wandb token\n", + "if wandb_token != \"\": \n", + " !wandb login {wandb_token}\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Step 6: Generate Config Files\n", + "\n", + "Run the following code block and the config files are automatically generated." + ] + }, + { + "cell_type": "code", + "execution_count": 78, + "metadata": { + "cellView": "form", + "id": "a1pK61X_VY2L" + }, + "outputs": [], + "source": [ + "# you can custom these strings to distiguish your different exps\n", + "exp_name = f\"sdxl_child_removal_v2\"\n", + "save_name = f\"{exp_name}\"\n", + "run_name = f\"{exp_name}\"\n", + "\n", + "config_file_path = f\"configs/{save_name}/config.yaml\"\n", + "prompts_file_path = f\"configs/{save_name}/prompt.yaml\"\n", + "\n", + "\n", + "config_file_content = f\"\"\"\n", + "prompts_file: \"{prompts_file_path}\"\n", + "\n", + "pretrained_model:\n", + " name_or_path: \"{pretrained_sd_model}\"\n", + " v2: {is_v2_model}\n", + " v_pred: {is_v_prediction_model}\n", + " clip_skip: {clip_skip}\n", + "\n", + "network:\n", + " rank: {dim}\n", + " alpha: 1.0\n", + "\n", + "train:\n", + " precision: {precision}\n", + " noise_scheduler: \"ddim\"\n", + " iterations: {iterations}\n", + " batch_size: {batch_size}\n", + " lr: {lr}\n", + " unet_lr: {lr}\n", + " text_encoder_lr: {0.5 * lr}\n", + " optimizer_type: \"{optimizer}\"\n", + " lr_scheduler: \"{lr_scheduler}\"\n", + " lr_warmup_steps: 500\n", + " lr_scheduler_num_cycles: 3\n", + " max_denoising_steps: {max_denoising_steps}\n", + "\n", + "save:\n", + " name: \"{save_name}\"\n", + " path: \"output/{save_name}\"\n", + " per_steps: {save_per_steps}\n", + " precision: {precision}\n", + "\n", + "logging:\n", + " use_wandb: \"false\"\n", + " interval: 500\n", + " seed: 0\n", + " generate_num: {generate_num}\n", + " run_name: \"{run_name}\"\n", + " verbose: {verbose}\n", + " prompts: {prompts_to_visualize}\n", + "\n", + "other:\n", + " use_xformers: true\n", + "\"\"\"\n", + "\n", + "import os\n", + "if not os.path.exists(f\"./configs/{save_name}\"):\n", + " os.makedirs(f\"./configs/{save_name}\")\n", + "\n", + "with open(config_file_path, \"w\") as f:\n", + " f.write(config_file_content)\n", + "\n", + "if multiple_concepts:\n", + " prompts_file_content = \"\" \n", + "\n", + " for target_concept, surrogate_concept in zip(target_concepts, surrogate_concepts):\n", + " prompts_file_content += f\"\"\"\n", + "- target: \"{target_concept}\"\n", + " positive: \"{target_concept}\"\n", + " unconditional: \"\"\n", + " neutral: \"{surrogate_concept}\"\n", + " action: \"{mode}\"\n", + " guidance_scale: \"{erasing_scale}\"\n", + " resolution: {resolution}\n", + " batch_size: {batch_size}\n", + " dynamic_resolution: {dynamic_resolution}\n", + " la_strength: {la_strength}\n", + " sampling_batch_size: {sampling_batch_size}\n", + " \"\"\"\n", + "else:\n", + " prompts_file_content = f\"\"\"\n", + "- target: \"{target_concept}\"\n", + " positive: \"{target_concept}\"\n", + " unconditional: \"\"\n", + " neutral: \"{surrogate_concept}\"\n", + " action: \"{mode}\"\n", + " guidance_scale: \"{erasing_scale}\"\n", + " resolution: {resolution}\n", + " batch_size: {batch_size}\n", + " dynamic_resolution: {dynamic_resolution}\n", + " la_strength: {la_strength}\n", + " sampling_batch_size: {sampling_batch_size}\n", + " \"\"\"\n", + "\n", + "with open(prompts_file_path, \"w\") as f:\n", + " f.write(prompts_file_content)\n" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "id": "wb9aX9BBVY2L" + }, + "source": [ + "## Start Training" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "WchSOvSHVY2L" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/workspace/SPM/./train_spm_xl_mem_reduce.py:41: PydanticDeprecatedSince20: The `json` method is deprecated; use `model_dump_json` instead. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.10/migration/\n", + " \"prompts\": \",\".join([prompt.json() for prompt in prompts]),\n", + "/workspace/SPM/./train_spm_xl_mem_reduce.py:42: PydanticDeprecatedSince20: The `json` method is deprecated; use `model_dump_json` instead. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.10/migration/\n", + " \"config\": config.json(),\n", + "Loading local checkpoint from /workspace/SPM/URPMXL-V6.safetensors\n", + "Fetching 17 files: 100%|█████████████████████| 17/17 [00:00<00:00, 42747.70it/s]\n", + "Loading pipeline components...: 0%| | 0/7 [00:00