main : fix session loading bug (#3400)

ggerganov · ggerganov · commit b8fe4b5cc9cb · 2023-10-11T23:55:41.000+03:00
diff --git a/examples/main/main.cpp b/examples/main/main.cpp
@@ -297,6 +297,9 @@ int main(int argc, char ** argv) {
             LOG_TEE("%s: session file matches %zu / %zu tokens of prompt\n",
                 __func__, n_matching_session_tokens, embd_inp.size());
         }
+
+        // remove any "future" tokens that we might have inherited from the previous session
+        llama_kv_cache_tokens_rm(ctx, n_matching_session_tokens, -1);
     }
 
     LOGLN(
@@ -545,9 +548,6 @@ int main(int argc, char ** argv) {
                 if (i > 0) {
                     embd.erase(embd.begin(), embd.begin() + i);
                 }
-
-                // remove any "future" tokens that we might have inherited from the session from the KV cache
-                llama_kv_cache_tokens_rm(ctx, n_past, -1);
             }
 
             // evaluate tokens in batches

Original file line number	Diff line number	Diff line change
`@@ -297,6 +297,9 @@ int main(int argc, char ** argv) {`
`297`	`297`	`LOG_TEE("%s: session file matches %zu / %zu tokens of prompt\n",`
`298`	`298`	`__func__, n_matching_session_tokens, embd_inp.size());`
`299`	`299`	`}`
	`300`	`+`
	`301`	`+ // remove any "future" tokens that we might have inherited from the previous session`
	`302`	`+ llama_kv_cache_tokens_rm(ctx, n_matching_session_tokens, -1);`
`300`	`303`	`}`
`301`	`304`
`302`	`305`	`LOGLN(`
`@@ -545,9 +548,6 @@ int main(int argc, char ** argv) {`
`545`	`548`	`if (i > 0) {`
`546`	`549`	`embd.erase(embd.begin(), embd.begin() + i);`
`547`	`550`	`}`
`548`		`-`
`549`		`- // remove any "future" tokens that we might have inherited from the session from the KV cache`
`550`		`- llama_kv_cache_tokens_rm(ctx, n_past, -1);`
`551`	`551`	`}`
`552`	`552`
`553`	`553`	`// evaluate tokens in batches`