| Copyright | (c) 2025 Tushar Adhatrao |
|---|---|
| License | MIT |
| Maintainer | Tushar Adhatrao <[email protected]> |
| Safe Haskell | None |
| Language | Haskell2010 |
Llama.KVCache
Description
Synopsis
- kvCacheViewInit :: Context -> Int -> IO LlamaKvCacheView
- kvSelfSeqAdd :: Context -> LlamaSeqId -> LlamaPos -> LlamaPos -> LlamaPos -> IO ()
- kvSelfSeqDiv :: Context -> LlamaSeqId -> LlamaPos -> LlamaPos -> Int -> IO ()
- kvSelfSeqPosMax :: Context -> LlamaSeqId -> IO LlamaPos
- kvSelfDefrag :: Context -> IO ()
- kvSelfCanShift :: Context -> IO Bool
- kvSelfUpdate :: Context -> IO ()
- kvSelfSeqKeep :: Context -> LlamaSeqId -> IO ()
- kvSelfSeqCopy :: Context -> LlamaSeqId -> LlamaSeqId -> LlamaPos -> LlamaPos -> IO ()
- kvSelfSeqRemove :: Context -> LlamaSeqId -> LlamaPos -> LlamaPos -> IO Bool
- kvSelfClear :: Context -> IO ()
- kvSelfUsedCells :: Context -> IO Int
- kvSelfNumTokens :: Context -> IO Int
- kvCacheViewUpdate :: Context -> Ptr LlamaKvCacheView -> IO ()
Documentation
kvCacheViewInit :: Context -> Int -> IO LlamaKvCacheView Source #
Convenience wrapper that allocates a LlamaKvCacheView and initializes it
kvSelfSeqAdd :: Context -> LlamaSeqId -> LlamaPos -> LlamaPos -> LlamaPos -> IO () Source #
Shift positions in a sequence by a delta.
kvSelfSeqDiv :: Context -> LlamaSeqId -> LlamaPos -> LlamaPos -> Int -> IO () Source #
Divide positions in a sequence by a factor (used for attention windowing).
kvSelfSeqPosMax :: Context -> LlamaSeqId -> IO LlamaPos Source #
Get the maximum position stored in the KV cache for a given sequence.
kvSelfDefrag :: Context -> IO () Source #
Defragment the KV cache to optimize memory usage.
kvSelfCanShift :: Context -> IO Bool Source #
Check whether the KV cache can be shifted (e.g., due to full buffer).
kvSelfUpdate :: Context -> IO () Source #
Update the KV cache's internal state (e.g., after manual modifications).
kvSelfSeqKeep :: Context -> LlamaSeqId -> IO () Source #
Prevent a sequence from being removed during KV cache cleanup.
kvSelfSeqCopy :: Context -> LlamaSeqId -> LlamaSeqId -> LlamaPos -> LlamaPos -> IO () Source #
Copy a range of positions from one sequence to another.
kvSelfSeqRemove :: Context -> LlamaSeqId -> LlamaPos -> LlamaPos -> IO Bool Source #
Remove a range of positions from a sequence.
kvSelfClear :: Context -> IO () Source #
Clear all sequences from the KV cache.
kvSelfNumTokens :: Context -> IO Int Source #
Get total number of tokens currently stored in the KV cache.
kvCacheViewUpdate :: Context -> Ptr LlamaKvCacheView -> IO () Source #
Update the KV cache view to reflect current state.