Skip to content

Commit e236528

Browse files
mofosynecompilade
andauthored
gguf_hash.py: Add sha256 (#8470)
* gguf_hash.py: Add sha256 * gguf_hash.py: rename string UUIDv5 --> uuid * Apply suggestions from code review Co-authored-by: compilade <[email protected]> --------- Co-authored-by: compilade <[email protected]>
1 parent fa79495 commit e236528

File tree

1 file changed

+19
-8
lines changed

1 file changed

+19
-8
lines changed

gguf-py/scripts/gguf_hash.py

Lines changed: 19 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,9 @@
2727

2828
# For more information about what field.parts and field.data represent,
2929
# please see the comments in the modify_gguf.py example.
30-
def gguf_hash(reader: GGUFReader, filename: str, disable_progress_bar) -> None:
30+
def gguf_hash(reader: GGUFReader, filename: str, disable_progress_bar: bool, no_layer: bool) -> None:
3131
sha1 = hashlib.sha1()
32+
sha256 = hashlib.sha256()
3233
uuidv5_sha1 = hashlib.sha1()
3334
uuidv5_sha1.update(UUID_NAMESPACE_LLAMA_CPP.bytes)
3435

@@ -50,7 +51,7 @@ def gguf_hash(reader: GGUFReader, filename: str, disable_progress_bar) -> None:
5051
bar = tqdm(desc="Hashing", total=total_weights, unit="weights", unit_scale=True, disable=disable_progress_bar)
5152

5253
# Hashing Process
53-
for n, tensor in enumerate(reader.tensors, 1):
54+
for tensor in reader.tensors:
5455

5556
# We don't need these
5657
if tensor.name.endswith((".attention.masked_bias", ".attention.bias", ".rotary_emb.inv_freq")):
@@ -62,29 +63,39 @@ def gguf_hash(reader: GGUFReader, filename: str, disable_progress_bar) -> None:
6263
sum_weights_in_tensor *= dim
6364
bar.update(sum_weights_in_tensor)
6465

65-
sha1_layer = hashlib.sha1()
66-
sha1_layer.update(tensor.data.data)
66+
if not no_layer:
67+
68+
sha1_layer = hashlib.sha1()
69+
sha1_layer.update(tensor.data.data)
70+
print("sha1 {0} {1}:{2}".format(sha1_layer.hexdigest(), filename, tensor.name)) # noqa: NP100
71+
72+
sha256_layer = hashlib.sha256()
73+
sha256_layer.update(tensor.data.data)
74+
print("sha256 {0} {1}:{2}".format(sha256_layer.hexdigest(), filename, tensor.name)) # noqa: NP100
75+
6776
sha1.update(tensor.data.data)
77+
sha256.update(tensor.data.data)
6878
uuidv5_sha1.update(tensor.data.data)
69-
print("sha1 {0} {1}:{2}".format(sha1_layer.hexdigest(), filename, tensor.name)) # noqa: NP100
7079

7180
# Flush Hash Progress Bar
7281
bar.close()
7382

7483
# Display Hash Output
75-
print("sha1 {0} {1}".format(sha1.hexdigest(), filename)) # noqa: NP100
76-
print("UUIDv5 {0} {1}".format(uuid.UUID(bytes=uuidv5_sha1.digest()[:16], version=5), filename)) # noqa: NP100
84+
print("sha1 {0} {1}".format(sha1.hexdigest(), filename)) # noqa: NP100
85+
print("sha256 {0} {1}".format(sha256.hexdigest(), filename)) # noqa: NP100
86+
print("uuid {0} {1}".format(uuid.UUID(bytes=uuidv5_sha1.digest()[:16], version=5), filename)) # noqa: NP100
7787

7888

7989
def main() -> None:
8090
parser = argparse.ArgumentParser(description="Dump GGUF file metadata")
8191
parser.add_argument("model", type=str, help="GGUF format model filename")
92+
parser.add_argument("--no-layer", action="store_true", help="exclude per layer hash")
8293
parser.add_argument("--verbose", action="store_true", help="increase output verbosity")
8394
parser.add_argument("--progressbar", action="store_true", help="enable progressbar")
8495
args = parser.parse_args(None if len(sys.argv) > 1 else ["--help"])
8596
logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO)
8697
reader = GGUFReader(args.model, 'r')
87-
gguf_hash(reader, args.model, not args.progressbar)
98+
gguf_hash(reader, args.model, not args.progressbar, args.no_layer)
8899

89100

90101
if __name__ == '__main__':

0 commit comments

Comments
 (0)