{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "%cd /content\n", "!apt install -qqy\n", "!pip install torchsde einops pyaml diffusers transformers peft accelerate aiohttp kornia spandrel sentencepiece ipywidgets termcolor moviepy==1.0.3\n", "\n", "!git clone https://github.com/comfyanonymous/ComfyUI /content/ComfyUI\n", "!git clone https://github.com/ltdrdata/ComfyUI-Manager /content/ComfyUI/custom_nodes/ComfyUI-Manager\n", "!git clone -b dev https://github.com/camenduru/ComfyUI-Fluxpromptenhancer /content/ComfyUI/custom_nodes/ComfyUI-Fluxpromptenhancer\n", "\n", "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/FLUX.1-dev/resolve/main/t5xxl_fp16.safetensors -d /content/ComfyUI/models/clip -o t5xxl_fp16.safetensors\n", "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/Lightricks/LTX-Video/resolve/main/ltx-video-2b-v0.9.safetensors -d /content/ComfyUI/models/checkpoints -o ltx-video-2b-v0.9.safetensors\n", "\n", "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/gokaygokay/Flux-Prompt-Enhance/raw/main/config.json -d /content/ComfyUI/models/LLM/Flux-Prompt-Enhance -o config.json\n", "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/gokaygokay/Flux-Prompt-Enhance/raw/main/generation_config.json -d /content/ComfyUI/models/LLM/Flux-Prompt-Enhance -o generation_config.json\n", "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/gokaygokay/Flux-Prompt-Enhance/resolve/main/model.safetensors -d /content/ComfyUI/models/LLM/Flux-Prompt-Enhance -o model.safetensors\n", "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/gokaygokay/Flux-Prompt-Enhance/raw/main/special_tokens_map.json -d /content/ComfyUI/models/LLM/Flux-Prompt-Enhance -o special_tokens_map.json\n", "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/gokaygokay/Flux-Prompt-Enhance/resolve/main/spiece.model -d /content/ComfyUI/models/LLM/Flux-Prompt-Enhance -o spiece.model\n", "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/gokaygokay/Flux-Prompt-Enhance/raw/main/tokenizer.json -d /content/ComfyUI/models/LLM/Flux-Prompt-Enhance -o tokenizer.json\n", "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/gokaygokay/Flux-Prompt-Enhance/raw/main/tokenizer_config.json -d /content/ComfyUI/models/LLM/Flux-Prompt-Enhance -o tokenizer_config.json" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "%cd /content/ComfyUI\n", "\n", "import os, shutil, json, requests, random, time\n", "from urllib.parse import urlsplit\n", "\n", "import torch\n", "from PIL import Image\n", "from moviepy.editor import ImageSequenceClip\n", "import numpy as np\n", "\n", "from nodes import NODE_CLASS_MAPPINGS, load_custom_node\n", "from comfy_extras import nodes_images, nodes_lt, nodes_custom_sampler\n", "\n", "load_custom_node(\"/content/ComfyUI/custom_nodes/ComfyUI-Fluxpromptenhancer\")\n", "\n", "FluxPromptEnhance = NODE_CLASS_MAPPINGS[\"FluxPromptEnhance\"]()\n", "CLIPLoader = NODE_CLASS_MAPPINGS[\"CLIPLoader\"]()\n", "CLIPTextEncode = NODE_CLASS_MAPPINGS[\"CLIPTextEncode\"]()\n", "LoadImage = NODE_CLASS_MAPPINGS[\"LoadImage\"]()\n", "CheckpointLoaderSimple = NODE_CLASS_MAPPINGS[\"CheckpointLoaderSimple\"]()\n", "LTXVImgToVideo = nodes_lt.NODE_CLASS_MAPPINGS[\"LTXVImgToVideo\"]()\n", "LTXVConditioning = nodes_lt.NODE_CLASS_MAPPINGS[\"LTXVConditioning\"]()\n", "SamplerCustom = nodes_custom_sampler.NODE_CLASS_MAPPINGS[\"SamplerCustom\"]()\n", "KSamplerSelect = nodes_custom_sampler.NODE_CLASS_MAPPINGS[\"KSamplerSelect\"]()\n", "LTXVScheduler = nodes_lt.NODE_CLASS_MAPPINGS[\"LTXVScheduler\"]()\n", "VAEDecode = NODE_CLASS_MAPPINGS[\"VAEDecode\"]()\n", "SaveAnimatedWEBP = nodes_images.NODE_CLASS_MAPPINGS[\"SaveAnimatedWEBP\"]()\n", "\n", "with torch.inference_mode():\n", " clip = CLIPLoader.load_clip(\"t5xxl_fp16.safetensors\", type=\"ltxv\")[0]\n", " model, _, vae = CheckpointLoaderSimple.load_checkpoint(\"ltx-video-2b-v0.9.safetensors\")\n", "\n", "def download_file(url, save_dir, file_name):\n", " os.makedirs(save_dir, exist_ok=True)\n", " file_suffix = os.path.splitext(urlsplit(url).path)[1]\n", " file_name_with_suffix = file_name + file_suffix\n", " file_path = os.path.join(save_dir, file_name_with_suffix)\n", " response = requests.get(url)\n", " response.raise_for_status()\n", " with open(file_path, 'wb') as file:\n", " file.write(response.content)\n", " return file_path\n", "\n", "def webp_to_mp4(input_webp, output_mp4, fps=10):\n", " with Image.open(input_webp) as img:\n", " frames = []\n", " try:\n", " while True:\n", " frame = img.copy()\n", " frames.append(frame)\n", " img.seek(img.tell() + 1)\n", " except EOFError:\n", " pass\n", " temp_dir = \"temp_frames\"\n", " os.makedirs(temp_dir, exist_ok=True)\n", " temp_frame_paths = []\n", " for i, frame in enumerate(frames):\n", " frame_path = os.path.join(temp_dir, f\"frame_{i:04d}.png\")\n", " frame.save(frame_path)\n", " temp_frame_paths.append(frame_path)\n", " clip = ImageSequenceClip(temp_frame_paths, fps=fps)\n", " clip.write_videofile(output_mp4, codec=\"libx264\", fps=fps)\n", " for path in temp_frame_paths:\n", " os.remove(path)\n", " os.rmdir(temp_dir)\n", "\n", "@torch.inference_mode()\n", "def generate(input):\n", " values = input[\"input\"]\n", "\n", " input_image = values['input_image']\n", " input_image = download_file(url=input_image, save_dir='/content/ComfyUI/input', file_name='input_image')\n", " positive_prompt = values['positive_prompt']\n", " negative_prompt = values['negative_prompt']\n", " frame_rate = values['frame_rate']\n", " sampler_name = values['sampler_name']\n", " steps = values['steps']\n", " max_shift = values['max_shift']\n", " base_shift = values['base_shift']\n", " stretch = values['stretch']\n", " terminal = values['terminal']\n", " width = values['width']\n", " height = values['height']\n", " length = values['length']\n", " add_noise = values['add_noise']\n", " noise_seed = values['noise_seed']\n", " cfg = values['cfg']\n", " fps = values['fps']\n", " prompt_enhance = values['prompt_enhance']\n", "\n", " if noise_seed == 0:\n", " random.seed(int(time.time()))\n", " noise_seed = random.randint(0, 18446744073709551615)\n", " print(noise_seed)\n", "\n", " if prompt_enhance:\n", " positive_prompt = FluxPromptEnhance.enhance_prompt(positive_prompt, noise_seed)[0]\n", " print(positive_prompt)\n", " conditioning_positive = CLIPTextEncode.encode(clip, positive_prompt)[0]\n", " conditioning_negative = CLIPTextEncode.encode(clip, negative_prompt)[0]\n", " image = LoadImage.load_image(input_image)[0]\n", " positive, negative, latent_image = LTXVImgToVideo.generate(conditioning_positive, conditioning_negative, image, vae, width, height, length, batch_size=1)\n", " positive, negative = LTXVConditioning.append(positive, negative, frame_rate)\n", " sampler = KSamplerSelect.get_sampler(sampler_name)[0]\n", " sigmas = LTXVScheduler.get_sigmas(steps, max_shift, base_shift, stretch, terminal, latent=None)[0]\n", " samples = SamplerCustom.sample(model, add_noise, noise_seed, cfg, positive, negative, sampler, sigmas, latent_image)[0]\n", " images = VAEDecode.decode(vae, samples)[0].detach()\n", " video = SaveAnimatedWEBP.save_images(images, fps, filename_prefix=f\"ltx-video-{noise_seed}-tost\", lossless=False, quality=90, method=\"default\")\n", " source = video['ui']['images'][0]['filename']\n", " source = f\"/content/ComfyUI/output/{source}\"\n", " destination = f\"/content/ltx-video-{noise_seed}-tost.webp\"\n", " shutil.move(source, destination)\n", " webp_to_mp4(f\"/content/ltx-video-{noise_seed}-tost.webp\", f\"/content/ltx-video-{noise_seed}-tost.mp4\", fps=fps)\n", " \n", " result = f\"/content/ltx-video-{noise_seed}-tost.mp4\"\n", "\n", " return result" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "input = { \n", " \"input\": {\n", " \"input_image\": \"https://files.catbox.moe/pidugn.png\",\n", " \"positive_prompt\": \"best quality, 4k, HDR, an inteview shot person talking\",\n", " \"negative_prompt\": \"low quality, worst quality, deformed, distorted, disfigured, motion smear, motion artifacts, fused fingers, bad anatomy, weird hand, ugly\",\n", " \"frame_rate\": 25,\n", " \"sampler_name\": \"euler\",\n", " \"steps\": 30,\n", " \"max_shift\": 2.05,\n", " \"base_shift\": 0.95,\n", " \"stretch\": True,\n", " \"terminal\": 0.10,\n", " \"width\": 768,\n", " \"height\": 512,\n", " \"length\": 97,\n", " \"add_noise\": True,\n", " \"noise_seed\": 0,\n", " \"cfg\": 3.0,\n", " \"fps\": 24,\n", " \"prompt_enhance\": True\n", " }\n", "}\n", "video = generate(input)\n", "from IPython.display import Video\n", "Video(video, embed=True)" ] } ], "metadata": { "kernelspec": { "display_name": "ComfyUI0-venv", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.12" } }, "nbformat": 4, "nbformat_minor": 2 }