Make default hash lib configurable without code changes via CLI argument (#3947)

* cli_args: Add --duplicate-check-hash-function.

* server.py: compare_image_hash configurable hash function

Uses an argument added in cli_args to specify the type of hashing to default to for duplicate hash checking.  Uses an `eval()` to identify the specific hashlib class to utilize, but ultimately safely operates because we have specific options and only those options/choices in the arg parser.  So we don't have any unsafe input there.

* Add hasher() to node_helpers

* hashlib selection moved to node_helpers

* default-hashing-function instead of dupe checking hasher

This makes a default-hashing-function option instead of previous selected option.

* Use args.default_hashing_function

* Use safer handling for node_helpers.hasher()

Uses a safer handling method than `eval` to evaluate default hashing function.

* Stray parentheses are evil.

* Indentation fix.

Somehow when I hit save I didn't notice I missed a space to make indentation work proper.  Oops!
This commit is contained in:
Thomas Ward 2024-07-16 18:27:09 -04:00 committed by GitHub
parent f2298799ba
commit c5a48b15bd
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 19 additions and 2 deletions

View File

@ -112,6 +112,7 @@ vram_group.add_argument("--lowvram", action="store_true", help="Split the unet i
vram_group.add_argument("--novram", action="store_true", help="When lowvram isn't enough.") vram_group.add_argument("--novram", action="store_true", help="When lowvram isn't enough.")
vram_group.add_argument("--cpu", action="store_true", help="To use the CPU for everything (slow).") vram_group.add_argument("--cpu", action="store_true", help="To use the CPU for everything (slow).")
parser.add_argument("--default-hashing-function", type=str, choices=['md5', 'sha1', 'sha256', 'sha512'], default='sha256', help="Allows you to choose the hash function to use for duplicate filename / contents comparison. Default is sha256.")
parser.add_argument("--disable-smart-memory", action="store_true", help="Force ComfyUI to agressively offload to regular ram instead of keeping models in vram when it can.") parser.add_argument("--disable-smart-memory", action="store_true", help="Force ComfyUI to agressively offload to regular ram instead of keeping models in vram when it can.")
parser.add_argument("--deterministic", action="store_true", help="Make pytorch use slower deterministic algorithms when it can. Note that this might not make images deterministic in all cases.") parser.add_argument("--deterministic", action="store_true", help="Make pytorch use slower deterministic algorithms when it can. Note that this might not make images deterministic in all cases.")

View File

@ -1,3 +1,7 @@
import hashlib
from comfy.cli_args import args
from PIL import ImageFile, UnidentifiedImageError from PIL import ImageFile, UnidentifiedImageError
def conditioning_set_values(conditioning, values={}): def conditioning_set_values(conditioning, values={}):
@ -22,3 +26,12 @@ def pillow(fn, arg):
if prev_value is not None: if prev_value is not None:
ImageFile.LOAD_TRUNCATED_IMAGES = prev_value ImageFile.LOAD_TRUNCATED_IMAGES = prev_value
return x return x
def hasher():
hashfuncs = {
"md5": hashlib.md5,
"sha1": hashlib.sha1,
"sha256": hashlib.sha256,
"sha512": hashlib.sha512
}
return hashfuncs[args.default_hashing_function]

View File

@ -25,6 +25,7 @@ import mimetypes
from comfy.cli_args import args from comfy.cli_args import args
import comfy.utils import comfy.utils
import comfy.model_management import comfy.model_management
import node_helpers
from app.frontend_management import FrontendManager from app.frontend_management import FrontendManager
from app.user_manager import UserManager from app.user_manager import UserManager
@ -161,10 +162,12 @@ class PromptServer():
return type_dir, dir_type return type_dir, dir_type
def compare_image_hash(filepath, image): def compare_image_hash(filepath, image):
hasher = node_helpers.hasher()
# function to compare hashes of two images to see if it already exists, fix to #3465 # function to compare hashes of two images to see if it already exists, fix to #3465
if os.path.exists(filepath): if os.path.exists(filepath):
a = hashlib.sha256() a = hasher()
b = hashlib.sha256() b = hasher()
with open(filepath, "rb") as f: with open(filepath, "rb") as f:
a.update(f.read()) a.update(f.read())
b.update(image.file.read()) b.update(image.file.read())