27
27
28
28
# For more information about what field.parts and field.data represent,
29
29
# please see the comments in the modify_gguf.py example.
30
- def gguf_hash (reader : GGUFReader , filename : str , disable_progress_bar ) -> None :
30
+ def gguf_hash (reader : GGUFReader , filename : str , disable_progress_bar : bool , no_layer : bool ) -> None :
31
31
sha1 = hashlib .sha1 ()
32
+ sha256 = hashlib .sha256 ()
32
33
uuidv5_sha1 = hashlib .sha1 ()
33
34
uuidv5_sha1 .update (UUID_NAMESPACE_LLAMA_CPP .bytes )
34
35
@@ -50,7 +51,7 @@ def gguf_hash(reader: GGUFReader, filename: str, disable_progress_bar) -> None:
50
51
bar = tqdm (desc = "Hashing" , total = total_weights , unit = "weights" , unit_scale = True , disable = disable_progress_bar )
51
52
52
53
# Hashing Process
53
- for n , tensor in enumerate ( reader .tensors , 1 ) :
54
+ for tensor in reader .tensors :
54
55
55
56
# We don't need these
56
57
if tensor .name .endswith ((".attention.masked_bias" , ".attention.bias" , ".rotary_emb.inv_freq" )):
@@ -62,29 +63,39 @@ def gguf_hash(reader: GGUFReader, filename: str, disable_progress_bar) -> None:
62
63
sum_weights_in_tensor *= dim
63
64
bar .update (sum_weights_in_tensor )
64
65
65
- sha1_layer = hashlib .sha1 ()
66
- sha1_layer .update (tensor .data .data )
66
+ if not no_layer :
67
+
68
+ sha1_layer = hashlib .sha1 ()
69
+ sha1_layer .update (tensor .data .data )
70
+ print ("sha1 {0} {1}:{2}" .format (sha1_layer .hexdigest (), filename , tensor .name )) # noqa: NP100
71
+
72
+ sha256_layer = hashlib .sha256 ()
73
+ sha256_layer .update (tensor .data .data )
74
+ print ("sha256 {0} {1}:{2}" .format (sha256_layer .hexdigest (), filename , tensor .name )) # noqa: NP100
75
+
67
76
sha1 .update (tensor .data .data )
77
+ sha256 .update (tensor .data .data )
68
78
uuidv5_sha1 .update (tensor .data .data )
69
- print ("sha1 {0} {1}:{2}" .format (sha1_layer .hexdigest (), filename , tensor .name )) # noqa: NP100
70
79
71
80
# Flush Hash Progress Bar
72
81
bar .close ()
73
82
74
83
# Display Hash Output
75
- print ("sha1 {0} {1}" .format (sha1 .hexdigest (), filename )) # noqa: NP100
76
- print ("UUIDv5 {0} {1}" .format (uuid .UUID (bytes = uuidv5_sha1 .digest ()[:16 ], version = 5 ), filename )) # noqa: NP100
84
+ print ("sha1 {0} {1}" .format (sha1 .hexdigest (), filename )) # noqa: NP100
85
+ print ("sha256 {0} {1}" .format (sha256 .hexdigest (), filename )) # noqa: NP100
86
+ print ("uuid {0} {1}" .format (uuid .UUID (bytes = uuidv5_sha1 .digest ()[:16 ], version = 5 ), filename )) # noqa: NP100
77
87
78
88
79
89
def main () -> None :
80
90
parser = argparse .ArgumentParser (description = "Dump GGUF file metadata" )
81
91
parser .add_argument ("model" , type = str , help = "GGUF format model filename" )
92
+ parser .add_argument ("--no-layer" , action = "store_true" , help = "exclude per layer hash" )
82
93
parser .add_argument ("--verbose" , action = "store_true" , help = "increase output verbosity" )
83
94
parser .add_argument ("--progressbar" , action = "store_true" , help = "enable progressbar" )
84
95
args = parser .parse_args (None if len (sys .argv ) > 1 else ["--help" ])
85
96
logging .basicConfig (level = logging .DEBUG if args .verbose else logging .INFO )
86
97
reader = GGUFReader (args .model , 'r' )
87
- gguf_hash (reader , args .model , not args .progressbar )
98
+ gguf_hash (reader , args .model , not args .progressbar , args . no_layer )
88
99
89
100
90
101
if __name__ == '__main__' :
0 commit comments