go-ollama-ollama-interesting-comment-reduces-by-one-line

This mapping has 1 all-to-all group: every node on one side corresponds to every node on the other, and none is deleted or inserted. Each member carries an all N:M badge; selecting one highlights all of its counterparts, in the tree and in the code view alike.

A painting is a person's account of this edit as text, recorded independently of the node mapping - a rendering often has several equally defensible answers where the mapping has one. Deletions and insertions carry no position on the opposite side in a painting, so that panel draws no caret for them.

Before…lines 1–762 unchanged (762 lines)
			return nil, fmt.Errorf("memory layout cannot be allocated with num_gpu = %v", s.options.NumGPU)
		}
765
		// Memory allocation failed even though we created a layout that we thought should
		// fit in available memory. This could happen if either our free memory reports
		// are incorrect or if available memory is changing between layout and allocation
		// time. Apply an exponential backoff to try to find the real amount of available
		// space.
		if backoff > 1 {
			slog.Warn("memory layout cannot be allocated", "memory", resp.Memory)
			return nil, errors.New("memory layout cannot be allocated")
		} else if backoff == 0 {
			backoff = 0.01
		} else {
			backoff *= 2
		}
779
		slog.Info("model layout did not fit, applying backoff", "backoff", fmt.Sprintf("%.2f", backoff))
	}
}
}
784
…lines 785–1794 unchanged (1010 lines)

After…lines 1–762 unchanged (762 lines)
			return nil, fmt.Errorf("memory layout cannot be allocated with num_gpu = %v", s.options.NumGPU)
		}
765
		// Memory allocation failed even though we created a layout that we thought should
		// fit in available memory. This could happen if either our free memory reports
		// are incorrect or if available memory is changing between layout and allocation
		// time. Apply a backoff to try to find the real amount of available space.
		if backoff > 1 {
			slog.Warn("memory layout cannot be allocated", "memory", resp.Memory)
			return nil, errors.New("memory layout cannot be allocated")
		} else {
			backoff += 0.1
		}
776
		slog.Info("model layout did not fit, applying backoff", "backoff", fmt.Sprintf("%.2f", backoff))
	}
}
}
781
s.loadRequest.GPULayers = gpuLayers
…lines 783–1791 unchanged (1009 lines)

Before…lines 1–762 unchanged (762 lines)
			return nil, fmt.Errorf("memory layout cannot be allocated with num_gpu = %v", s.options.NumGPU)
		}
765
		// Memory allocation failed even though we created a layout that we thought should
		// fit in available memory. This could happen if either our free memory reports
		// are incorrect or if available memory is changing between layout and allocation
		// time. Apply an exponential backoff to try to find the real amount of available
		// space.
		if backoff > 1 {
			slog.Warn("memory layout cannot be allocated", "memory", resp.Memory)
			return nil, errors.New("memory layout cannot be allocated")
		} else if backoff == 0 {
			backoff = 0.01
		} else {
			backoff *= 2
		}
779
		slog.Info("model layout did not fit, applying backoff", "backoff", fmt.Sprintf("%.2f", backoff))
	}
}
}
784
…lines 785–1794 unchanged (1010 lines)

After…lines 1–762 unchanged (762 lines)
			return nil, fmt.Errorf("memory layout cannot be allocated with num_gpu = %v", s.options.NumGPU)
		}
765
		// Memory allocation failed even though we created a layout that we thought should
		// fit in available memory. This could happen if either our free memory reports
		// are incorrect or if available memory is changing between layout and allocation
		// time. Apply a backoff to try to find the real amount of available space.
		if backoff > 1 {
			slog.Warn("memory layout cannot be allocated", "memory", resp.Memory)
			return nil, errors.New("memory layout cannot be allocated")
		} else {
			backoff += 0.1
		}
776
		slog.Info("model layout did not fit, applying backoff", "backoff", fmt.Sprintf("%.2f", backoff))
	}
}
}
781
s.loadRequest.GPULayers = gpuLayers
…lines 783–1791 unchanged (1009 lines)

Before…lines 1–762 unchanged (762 lines)
			return nil, fmt.Errorf("memory layout cannot be allocated with num_gpu = %v", s.options.NumGPU)
		}
765
		// Memory allocation failed even though we created a layout that we thought should
		// fit in available memory. This could happen if either our free memory reports
		// are incorrect or if available memory is changing between layout and allocation
		// time. Apply an exponential backoff to try to find the real amount of available
		// space.
		if backoff > 1 {
			slog.Warn("memory layout cannot be allocated", "memory", resp.Memory)
			return nil, errors.New("memory layout cannot be allocated")
		} else if backoff == 0 {
			backoff = 0.01
		} else {
			backoff *= 2
		}
779
		slog.Info("model layout did not fit, applying backoff", "backoff", fmt.Sprintf("%.2f", backoff))
	}
}
}
784
…lines 785–1794 unchanged (1010 lines)

After…lines 1–762 unchanged (762 lines)
			return nil, fmt.Errorf("memory layout cannot be allocated with num_gpu = %v", s.options.NumGPU)
		}
765
		// Memory allocation failed even though we created a layout that we thought should
		// fit in available memory. This could happen if either our free memory reports
		// are incorrect or if available memory is changing between layout and allocation
		// time. Apply a backoff to try to find the real amount of available space.
		if backoff > 1 {
			slog.Warn("memory layout cannot be allocated", "memory", resp.Memory)
			return nil, errors.New("memory layout cannot be allocated")
		} else {
			backoff += 0.1
		}
776
		slog.Info("model layout did not fit, applying backoff", "backoff", fmt.Sprintf("%.2f", backoff))
	}
}
}
781
s.loadRequest.GPULayers = gpuLayers
…lines 783–1791 unchanged (1009 lines)

Before

source_file

package_clause

package "package"

package_identifier "llm"

import_declaration (+160 nodes collapsed)

type_declaration

type "type"

type_spec

type_identifier "filteredEnv"

slice_type

[ "["

] "]"

type_identifier "string"

method_declaration (+278 nodes collapsed)

type_declaration (+255 nodes collapsed)

comment "// llmServer is an instance of a runner hosting a single mod"...

type_declaration (+94 nodes collapsed)

type_declaration (+32 nodes collapsed)

type_declaration

type "type"

type_spec

type_identifier "ollamaServer"

struct_type

struct "struct"

field_declaration_list

{ "{"

field_declaration

type_identifier "llmServer"

field_declaration

field_identifier "mem"

pointer_type

* "*"

qualified_type

package_identifier "ml"

. "."

type_identifier "BackendMemory"

} "}"

comment "// LoadModel will load a model from disk. The model must be "...

comment "//"

comment "// It collects array values for arrays with a size less than"...

comment "// maxArraySize. If maxArraySize is 0, the default value of "...

comment "// the maxArraySize is negative, all arrays are collected."

function_declaration (+129 nodes collapsed)

comment "// NewLlamaServer will run a server for the given GPUs"

function_declaration (+1342 nodes collapsed)

function_declaration (+1141 nodes collapsed)

method_declaration (+26 nodes collapsed)

type_declaration

type "type"

type_spec

type_identifier "LoadOperation"

type_identifier "int"

comment "// The order of these constants are significant because we i"...

comment "// should be in order of increasingly loading the model."

const_declaration

const "const"

( "("

const_spec

identifier "LoadOperationFit"

type_identifier "LoadOperation"

= "="

expression_list

iota "iota"

comment "// Return memory requirements but do not allocate"

const_spec

identifier "LoadOperationAlloc"

comment "// Allocate memory but do not load the weights"

const_spec

identifier "LoadOperationCommit"

comment "// Load weights - further changes cannot be made after this"

const_spec

identifier "LoadOperationClose"

comment "// Close model and free memory"

) ")"

method_declaration (+85 nodes collapsed)

type_declaration (+55 nodes collapsed)

type_declaration

type "type"

type_spec

type_identifier "LoadResponse"

struct_type

struct "struct"

field_declaration_list

{ "{"

field_declaration

field_identifier "Success"

type_identifier "bool"

field_declaration

field_identifier "Memory"

qualified_type

package_identifier "ml"

. "."

type_identifier "BackendMemory"

} "}"

var_declaration

var "var"

var_spec

identifier "ErrLoadRequiredFull"

= "="

expression_list

call_expression

selector_expression

identifier "errors"

. "."

field_identifier "New"

argument_list

( "("

interpreted_string_literal

" "\""

interpreted_string_literal_content "unable to load full model on GPU"

" "\""

) ")"

method_declaration (+1546 nodes collapsed)

comment "// createGPULayers maps from the tensor splits assigned by t"...

comment "// of particular layers onto GPUs"

function_declaration (+426 nodes collapsed)

comment "// Load finds the optimal layout of layers to offload on GPU"...

comment "// It does this by:"

comment "// 1. Assigning the full model to the GPU with the largest a"...

comment "// 2. Attempting to allocate the layout and receiving the me"...

comment "// 3. Creating a new layout based on the updated memory info"...

comment "// 4. Going back to step 2 and looping until we either stabi"...

comment "//"

comment "// This process is repeated for higher levels of loading the"...

comment "// allowing for faster iteration, but may return less inform"...

comment "//"

comment "// Returns the list of GPU IDs that were used in the final a"...

method_declaration

func "func"

parameter_list

( "("

parameter_declaration

identifier "s"

pointer_type

* "*"

type_identifier "ollamaServer"

) ")"

field_identifier "Load"

parameter_list (+30 nodes collapsed)

parameter_list

( "("

parameter_declaration

slice_type

[ "["

] "]"

qualified_type

package_identifier "ml"

. "."

type_identifier "DeviceID"

, ","

parameter_declaration

type_identifier "error"

) ")"

block

{ "{"

statement_list

var_declaration

var "var"

var_spec

identifier "success"

type_identifier "bool"

defer_statement (+71 nodes collapsed)

expression_statement (+36 nodes collapsed)

short_var_declaration

expression_list

identifier "systemTotalMemory"

:= ":="

expression_list

selector_expression

identifier "systemInfo"

. "."

field_identifier "TotalMemory"

short_var_declaration

expression_list

identifier "systemFreeMemory"

:= ":="

expression_list

selector_expression

identifier "systemInfo"

. "."

field_identifier "FreeMemory"

short_var_declaration

expression_list

identifier "systemSwapFreeMemory"

:= ":="

expression_list

selector_expression

identifier "systemInfo"

. "."

field_identifier "FreeSwap"

expression_statement (+58 nodes collapsed)

for_statement (+189 nodes collapsed)

short_var_declaration

expression_list

identifier "pastAllocations"

:= ":="

expression_list

call_expression

identifier "make"

argument_list

( "("

map_type

map "map"

[ "["

type_identifier "uint64"

] "]"

struct_type

struct "struct"

field_declaration_list

{ "{"

} "}"

) ")"

var_declaration

var "var"

var_spec

identifier "backoff"

type_identifier "float32"

short_var_declaration (+27 nodes collapsed)

if_statement

if "if"

binary_expression

identifier "err"

!= "!="

nil "nil"

block

{ "{"

statement_list

return_statement

return "return"

expression_list

nil "nil"

, ","

identifier "err"

} "}"

if_statement (+31 nodes collapsed)

labeled_statement

label_name "nextOperation"

: ":"

for_statement

for "for"

for_clause

short_var_declaration

expression_list

identifier "operation"

:= ":="

expression_list

identifier "LoadOperationFit"

; ";"

binary_expression

identifier "operation"

< "<"

identifier "LoadOperationCommit"

; ";"

inc_statement

identifier "operation"

++ "++"

block

{ "{"

statement_list

labeled_statement

label_name "nextLoad"

: ":"

for_statement

for "for"

block

{ "{"

statement_list

assignment_statement

expression_list

selector_expression

identifier "s"

. "."

field_identifier "loadRequest"

. "."

field_identifier "GPULayers"

= "="

expression_list

identifier "gpuLayers"

short_var_declaration (+23 nodes collapsed)

if_statement

if "if"

binary_expression

identifier "err"

!= "!="

nil "nil"

block

{ "{"

statement_list

return_statement

return "return"

expression_list

nil "nil"

, ","

identifier "err"

} "}"

expression_statement

call_expression

selector_expression

identifier "resp"

. "."

field_identifier "Memory"

. "."

field_identifier "Log"

argument_list

( "("

selector_expression

identifier "slog"

. "."

field_identifier "LevelDebug"

) ")"

expression_statement (+33 nodes collapsed)

assignment_statement (+25 nodes collapsed)

assignment_statement

expression_list

selector_expression

identifier "s"

. "."

field_identifier "mem"

= "="

expression_list

unary_expression

& "&"

selector_expression

identifier "resp"

. "."

field_identifier "Memory"

for_statement

for "for"

block

{ "{"

statement_list

short_var_declaration (+27 nodes collapsed)

if_statement

if "if"

binary_expression

identifier "err"

!= "!="

nil "nil"

block

{ "{"

statement_list

return_statement

return "return"

expression_list

nil "nil"

, ","

identifier "err"

} "}"

expression_statement

call_expression

selector_expression

identifier "slog"

. "."

field_identifier "Debug"

argument_list

( "("

interpreted_string_literal

" "\""

interpreted_string_literal_content "new layout created"

" "\""

, ","

interpreted_string_literal

" "\""

interpreted_string_literal_content "layers"

" "\""

, ","

identifier "newGPULayers"

) ")"

comment "// We get additional memory information over time, which wil"...

comment "// layers that can fit, so fewer layers is actually better. "...

comment "// this layout before and it doesn't have more layers than t"...

comment "// trying to see if we can do better."

if_statement (+58 nodes collapsed)

comment "// If we are looping around a few different layouts due to g"...

comment "// GPUs, make sure that we try out the intermediate states. "...

comment "// looping between offloading 39 and 41 layers, we should al"...

comment "//"

comment "// This switches strategies to force an incremental number o"...

comment "// and checking the memory layout. If the allocation succeed"...

comment "// without forcing offload yields the same or greater number"...

comment "// the trial is successful."

comment "//"

comment "// This alternate strategy does not introduce the possibilit"...

comment "// state machine, as it exits this code block either with a "...

comment "// to the next operation or the original number of layers of"...

if_statement (+404 nodes collapsed)

comment "// If we generated a layout a second time or go backwards, t"...

comment "// layout before the repeat, which is already allocated."

if_statement

if "if"

selector_expression

identifier "resp"

. "."

field_identifier "Success"

block

{ "{"

statement_list

continue_statement

continue "continue"

label_name "nextOperation"

} "}"

if_statement (+41 nodes collapsed)

comment "// Memory allocation failed even though we created a layout "...

comment "// fit in available memory. This could happen if either our "...

comment "// are incorrect or if available memory is changing between "...

comment "// time. Apply an exponential backoff to try to find the rea"... all 2:1

comment "// space." all 2:1

if_statement

if "if"

binary_expression

identifier "backoff"

> ">"

int_literal "1"

block (+44 nodes collapsed)

else "else"

if_statement

if "if"

binary_expression

identifier "backoff"

== "=="

int_literal "0"

block

{ "{"

statement_list

assignment_statement

expression_list

identifier "backoff"

= "="

expression_list

float_literal "0.01"

} "}"

else "else"

block

{ "{"

statement_list

assignment_statement

expression_list

identifier "backoff"

*= "*="

expression_list

int_literal "2"

} "}"

expression_statement (+33 nodes collapsed)

} "}"

assignment_statement

expression_list

selector_expression

identifier "s"

. "."

field_identifier "loadRequest"

. "."

field_identifier "GPULayers"

= "="

expression_list

identifier "gpuLayers"

short_var_declaration (+23 nodes collapsed)

if_statement

if "if"

binary_expression

identifier "err"

!= "!="

nil "nil"

block

{ "{"

statement_list

return_statement

return "return"

expression_list

nil "nil"

, ","

identifier "err"

} "}"

assignment_statement

expression_list

identifier "success"

= "="

expression_list

selector_expression

identifier "resp"

. "."

field_identifier "Success"

assignment_statement

expression_list

selector_expression

identifier "s"

. "."

field_identifier "mem"

= "="

expression_list

unary_expression

& "&"

selector_expression

identifier "resp"

. "."

field_identifier "Memory"

if_statement (+49 nodes collapsed)

return_statement

return "return"

expression_list

call_expression

identifier "uniqueDeviceIDs"

argument_list

( "("

identifier "gpuLayers"

) ")"

, ","

nil "nil"

} "}"

function_declaration (+121 nodes collapsed)

comment "// createLayout uses the current best view of memory require"...

comment "// It does this by:"

comment "// - Calculating how much space each layer requires"

comment "// - Calculating how much space each GPU has available for l"...

comment "// - Assigning layers"

comment "// - Ensuring that we don't exceed limits, such as requireme"...

method_declaration (+224 nodes collapsed)

method_declaration (+795 nodes collapsed)

comment "// verifyLayout ensures that we don't exceed limits, such as"...

method_declaration (+551 nodes collapsed)

comment "// assignLayers packs the maximum number of layers onto the "...

function_declaration (+279 nodes collapsed)

comment "// findBestFit binary searches to find the smallest capacity"...

comment "// the max number of layers. The capacity factor is multipli"...

comment "// each GPU and a small one will force even balancing."

function_declaration (+212 nodes collapsed)

comment "// greedyFit assigns layers incrementally to GPUs, spilling "...

function_declaration (+328 nodes collapsed)

comment "// waitUntilRunnerLaunched sleeps until the runner subproces"...

comment "// to respond to status requests"

method_declaration (+126 nodes collapsed)

comment "// initModel sends a load request to the runner based on the"...

comment "// and parameters"

method_declaration (+412 nodes collapsed)

type_declaration

type "type"

type_spec

type_identifier "ServerStatus"

type_identifier "int"

const_declaration (+21 nodes collapsed)

method_declaration (+98 nodes collapsed)

type_declaration (+23 nodes collapsed)

method_declaration (+616 nodes collapsed)

comment "// getServerStatusRetry will retry if ServerStatusNoSlotsAva"...

method_declaration (+141 nodes collapsed)

method_declaration (+79 nodes collapsed)

method_declaration (+685 nodes collapsed)

method_declaration (+63 nodes collapsed)

method_declaration (+26 nodes collapsed)

method_declaration (+71 nodes collapsed)

var_declaration

var "var"

var_spec

identifier "grammarJSON"

= "="

expression_list

raw_string_literal

` "`"

raw_string_literal_content "\nroot ::= object\nvalue ::= object | array | string | numb"...

` "`"

const_declaration

const "const"

const_spec

identifier "maxBufferSize"

= "="

expression_list

binary_expression

int_literal "512"

* "*"

selector_expression

identifier "format"

. "."

field_identifier "KiloByte"

type_declaration (+26 nodes collapsed)

type_declaration (+42 nodes collapsed)

comment "// DoneReason represents the reason why a completion respons"...

type_declaration

type "type"

type_spec

type_identifier "DoneReason"

type_identifier "int"

const_declaration

const "const"

( "("

comment "// DoneReasonStop indicates the completion stopped naturally"

const_spec

identifier "DoneReasonStop"

type_identifier "DoneReason"

= "="

expression_list

iota "iota"

comment "// DoneReasonLength indicates the completion stopped due to "...

const_spec

identifier "DoneReasonLength"

comment "// DoneReasonConnectionClosed indicates the completion stopp"...

const_spec

identifier "DoneReasonConnectionClosed"

) ")"

method_declaration (+59 nodes collapsed)

type_declaration (+64 nodes collapsed)

method_declaration (+1510 nodes collapsed)

type_declaration

type "type"

type_spec

type_identifier "EmbeddingRequest"

struct_type

struct "struct"

field_declaration_list

{ "{"

field_declaration

field_identifier "Content"

type_identifier "string"

raw_string_literal

` "`"

raw_string_literal_content "json:\"content\""

` "`"

} "}"

type_declaration

type "type"

type_spec

type_identifier "EmbeddingResponse"

struct_type

struct "struct"

field_declaration_list

{ "{"

field_declaration

field_identifier "Embedding"

slice_type

[ "["

] "]"

type_identifier "float32"

raw_string_literal

` "`"

raw_string_literal_content "json:\"embedding\""

` "`"

} "}"

method_declaration (+602 nodes collapsed)

type_declaration

type "type"

type_spec

type_identifier "TokenizeRequest"

struct_type

struct "struct"

field_declaration_list

{ "{"

field_declaration

field_identifier "Content"

type_identifier "string"

raw_string_literal

` "`"

raw_string_literal_content "json:\"content\""

` "`"

} "}"

type_declaration

type "type"

type_spec

type_identifier "TokenizeResponse"

struct_type

struct "struct"

field_declaration_list

{ "{"

field_declaration

field_identifier "Tokens"

slice_type

[ "["

] "]"

type_identifier "int"

raw_string_literal

` "`"

raw_string_literal_content "json:\"tokens\""

` "`"

} "}"

method_declaration (+220 nodes collapsed)

type_declaration

type "type"

type_spec

type_identifier "DetokenizeRequest"

struct_type

struct "struct"

field_declaration_list

{ "{"

field_declaration

field_identifier "Tokens"

slice_type

[ "["

] "]"

type_identifier "int"

raw_string_literal

` "`"

raw_string_literal_content "json:\"tokens\""

` "`"

} "}"

type_declaration

type "type"

type_spec

type_identifier "DetokenizeResponse"

struct_type

struct "struct"

field_declaration_list

{ "{"

field_declaration

field_identifier "Content"

type_identifier "string"

raw_string_literal

` "`"

raw_string_literal_content "json:\"content\""

` "`"

} "}"

method_declaration (+245 nodes collapsed)

method_declaration (+234 nodes collapsed)

method_declaration (+29 nodes collapsed)

method_declaration (+94 nodes collapsed)

method_declaration (+48 nodes collapsed)

method_declaration (+191 nodes collapsed)

method_declaration (+104 nodes collapsed)

method_declaration (+90 nodes collapsed)

method_declaration (+110 nodes collapsed)

After

source_file

package_clause

package "package"

package_identifier "llm"

import_declaration (+160 nodes collapsed)

type_declaration

type "type"

type_spec

type_identifier "filteredEnv"

slice_type

[ "["

] "]"

type_identifier "string"

method_declaration (+278 nodes collapsed)

type_declaration (+255 nodes collapsed)

comment "// llmServer is an instance of a runner hosting a single mod"...

type_declaration (+94 nodes collapsed)

type_declaration (+32 nodes collapsed)

type_declaration

type "type"

type_spec

type_identifier "ollamaServer"

struct_type

struct "struct"

field_declaration_list

{ "{"

field_declaration

type_identifier "llmServer"

field_declaration

field_identifier "mem"

pointer_type

* "*"

qualified_type

package_identifier "ml"

. "."

type_identifier "BackendMemory"

} "}"

comment "// LoadModel will load a model from disk. The model must be "...

comment "//"

comment "// It collects array values for arrays with a size less than"...

comment "// maxArraySize. If maxArraySize is 0, the default value of "...

comment "// the maxArraySize is negative, all arrays are collected."

function_declaration (+129 nodes collapsed)

comment "// NewLlamaServer will run a server for the given GPUs"

function_declaration (+1342 nodes collapsed)

function_declaration (+1141 nodes collapsed)

method_declaration (+26 nodes collapsed)

type_declaration

type "type"

type_spec

type_identifier "LoadOperation"

type_identifier "int"

comment "// The order of these constants are significant because we i"...

comment "// should be in order of increasingly loading the model."

const_declaration

const "const"

( "("

const_spec

identifier "LoadOperationFit"

type_identifier "LoadOperation"

= "="

expression_list

iota "iota"

comment "// Return memory requirements but do not allocate"

const_spec

identifier "LoadOperationAlloc"

comment "// Allocate memory but do not load the weights"

const_spec

identifier "LoadOperationCommit"

comment "// Load weights - further changes cannot be made after this"

const_spec

identifier "LoadOperationClose"

comment "// Close model and free memory"

) ")"

method_declaration (+85 nodes collapsed)

type_declaration (+55 nodes collapsed)

type_declaration

type "type"

type_spec

type_identifier "LoadResponse"

struct_type

struct "struct"

field_declaration_list

{ "{"

field_declaration

field_identifier "Success"

type_identifier "bool"

field_declaration

field_identifier "Memory"

qualified_type

package_identifier "ml"

. "."

type_identifier "BackendMemory"

} "}"

var_declaration

var "var"

var_spec

identifier "ErrLoadRequiredFull"

= "="

expression_list

call_expression

selector_expression

identifier "errors"

. "."

field_identifier "New"

argument_list

( "("

interpreted_string_literal

" "\""

interpreted_string_literal_content "unable to load full model on GPU"

" "\""

) ")"

method_declaration (+1546 nodes collapsed)

comment "// createGPULayers maps from the tensor splits assigned by t"...

comment "// of particular layers onto GPUs"

function_declaration (+426 nodes collapsed)

comment "// Load finds the optimal layout of layers to offload on GPU"...

comment "// It does this by:"

comment "// 1. Assigning the full model to the GPU with the largest a"...

comment "// 2. Attempting to allocate the layout and receiving the me"...

comment "// 3. Creating a new layout based on the updated memory info"...

comment "// 4. Going back to step 2 and looping until we either stabi"...

comment "//"

comment "// This process is repeated for higher levels of loading the"...

comment "// allowing for faster iteration, but may return less inform"...

comment "//"

comment "// Returns the list of GPU IDs that were used in the final a"...

method_declaration

func "func"

parameter_list

( "("

parameter_declaration

identifier "s"

pointer_type

* "*"

type_identifier "ollamaServer"

) ")"

field_identifier "Load"

parameter_list (+30 nodes collapsed)

parameter_list

( "("

parameter_declaration

slice_type

[ "["

] "]"

qualified_type

package_identifier "ml"

. "."

type_identifier "DeviceID"

, ","

parameter_declaration

type_identifier "error"

) ")"

block

{ "{"

statement_list

var_declaration

var "var"

var_spec

identifier "success"

type_identifier "bool"

defer_statement (+71 nodes collapsed)

expression_statement (+36 nodes collapsed)

short_var_declaration

expression_list

identifier "systemTotalMemory"

:= ":="

expression_list

selector_expression

identifier "systemInfo"

. "."

field_identifier "TotalMemory"

short_var_declaration

expression_list

identifier "systemFreeMemory"

:= ":="

expression_list

selector_expression

identifier "systemInfo"

. "."

field_identifier "FreeMemory"

short_var_declaration

expression_list

identifier "systemSwapFreeMemory"

:= ":="

expression_list

selector_expression

identifier "systemInfo"

. "."

field_identifier "FreeSwap"

expression_statement (+58 nodes collapsed)

for_statement (+189 nodes collapsed)

short_var_declaration

expression_list

identifier "pastAllocations"

:= ":="

expression_list

call_expression

identifier "make"

argument_list

( "("

map_type

map "map"

[ "["

type_identifier "uint64"

] "]"

struct_type

struct "struct"

field_declaration_list

{ "{"

} "}"

) ")"

var_declaration

var "var"

var_spec

identifier "backoff"

type_identifier "float32"

short_var_declaration (+27 nodes collapsed)

if_statement

if "if"

binary_expression

identifier "err"

!= "!="

nil "nil"

block

{ "{"

statement_list

return_statement

return "return"

expression_list

nil "nil"

, ","

identifier "err"

} "}"

if_statement (+31 nodes collapsed)

labeled_statement

label_name "nextOperation"

: ":"

for_statement

for "for"

for_clause

short_var_declaration

expression_list

identifier "operation"

:= ":="

expression_list

identifier "LoadOperationFit"

; ";"

binary_expression

identifier "operation"

< "<"

identifier "LoadOperationCommit"

; ";"

inc_statement

identifier "operation"

++ "++"

block

{ "{"

statement_list

labeled_statement

label_name "nextLoad"

: ":"

for_statement

for "for"

block

{ "{"

statement_list

assignment_statement

expression_list

selector_expression

identifier "s"

. "."

field_identifier "loadRequest"

. "."

field_identifier "GPULayers"

= "="

expression_list

identifier "gpuLayers"

short_var_declaration (+23 nodes collapsed)

if_statement

if "if"

binary_expression

identifier "err"

!= "!="

nil "nil"

block

{ "{"

statement_list

return_statement

return "return"

expression_list

nil "nil"

, ","

identifier "err"

} "}"

expression_statement

call_expression

selector_expression

identifier "resp"

. "."

field_identifier "Memory"

. "."

field_identifier "Log"

argument_list

( "("

selector_expression

identifier "slog"

. "."

field_identifier "LevelDebug"

) ")"

expression_statement (+33 nodes collapsed)

assignment_statement (+25 nodes collapsed)

assignment_statement

expression_list

selector_expression

identifier "s"

. "."

field_identifier "mem"

= "="

expression_list

unary_expression

& "&"

selector_expression

identifier "resp"

. "."

field_identifier "Memory"

for_statement

for "for"

block

{ "{"

statement_list

short_var_declaration (+27 nodes collapsed)

if_statement

if "if"

binary_expression

identifier "err"

!= "!="

nil "nil"

block

{ "{"

statement_list

return_statement

return "return"

expression_list

nil "nil"

, ","

identifier "err"

} "}"

expression_statement

call_expression

selector_expression

identifier "slog"

. "."

field_identifier "Debug"

argument_list

( "("

interpreted_string_literal

" "\""

interpreted_string_literal_content "new layout created"

" "\""

, ","

interpreted_string_literal

" "\""

interpreted_string_literal_content "layers"

" "\""

, ","

identifier "newGPULayers"

) ")"

comment "// We get additional memory information over time, which wil"...

comment "// layers that can fit, so fewer layers is actually better. "...

comment "// this layout before and it doesn't have more layers than t"...

comment "// trying to see if we can do better."

if_statement (+58 nodes collapsed)

comment "// If we are looping around a few different layouts due to g"...

comment "// GPUs, make sure that we try out the intermediate states. "...

comment "// looping between offloading 39 and 41 layers, we should al"...

comment "//"

comment "// This switches strategies to force an incremental number o"...

comment "// and checking the memory layout. If the allocation succeed"...

comment "// without forcing offload yields the same or greater number"...

comment "// the trial is successful."

comment "//"

comment "// This alternate strategy does not introduce the possibilit"...

comment "// state machine, as it exits this code block either with a "...

comment "// to the next operation or the original number of layers of"...

if_statement (+404 nodes collapsed)

comment "// If we generated a layout a second time or go backwards, t"...

comment "// layout before the repeat, which is already allocated."

if_statement

if "if"

selector_expression

identifier "resp"

. "."

field_identifier "Success"

block

{ "{"

statement_list

continue_statement

continue "continue"

label_name "nextOperation"

} "}"

if_statement (+41 nodes collapsed)

comment "// Memory allocation failed even though we created a layout "...

comment "// fit in available memory. This could happen if either our "...

comment "// are incorrect or if available memory is changing between "...

comment "// time. Apply a backoff to try to find the real amount of a"... all 2:1

if_statement

if "if"

binary_expression

identifier "backoff"

> ">"

int_literal "1"

block (+44 nodes collapsed)

else "else"

block

{ "{"

statement_list

assignment_statement

expression_list

identifier "backoff"

+= "+="

expression_list

float_literal "0.1"

} "}"

expression_statement (+33 nodes collapsed)

} "}"

assignment_statement

expression_list

selector_expression

identifier "s"

. "."

field_identifier "loadRequest"

. "."

field_identifier "GPULayers"

= "="

expression_list

identifier "gpuLayers"

short_var_declaration (+23 nodes collapsed)

if_statement

if "if"

binary_expression

identifier "err"

!= "!="

nil "nil"

block

{ "{"

statement_list

return_statement

return "return"

expression_list

nil "nil"

, ","

identifier "err"

} "}"

assignment_statement

expression_list

identifier "success"

= "="

expression_list

selector_expression

identifier "resp"

. "."

field_identifier "Success"

assignment_statement

expression_list

selector_expression

identifier "s"

. "."

field_identifier "mem"

= "="

expression_list

unary_expression

& "&"

selector_expression

identifier "resp"

. "."

field_identifier "Memory"

if_statement (+49 nodes collapsed)

return_statement

return "return"

expression_list

call_expression

identifier "uniqueDeviceIDs"

argument_list

( "("

identifier "gpuLayers"

) ")"

, ","

nil "nil"

} "}"

function_declaration (+121 nodes collapsed)

comment "// createLayout uses the current best view of memory require"...

comment "// It does this by:"

comment "// - Calculating how much space each layer requires"

comment "// - Calculating how much space each GPU has available for l"...

comment "// - Assigning layers"

comment "// - Ensuring that we don't exceed limits, such as requireme"...

method_declaration (+224 nodes collapsed)

method_declaration (+795 nodes collapsed)

comment "// verifyLayout ensures that we don't exceed limits, such as"...

method_declaration (+551 nodes collapsed)

comment "// assignLayers packs the maximum number of layers onto the "...

function_declaration (+279 nodes collapsed)

comment "// findBestFit binary searches to find the smallest capacity"...

comment "// the max number of layers. The capacity factor is multipli"...

comment "// each GPU and a small one will force even balancing."

function_declaration (+212 nodes collapsed)

comment "// greedyFit assigns layers incrementally to GPUs, spilling "...

function_declaration (+328 nodes collapsed)

comment "// waitUntilRunnerLaunched sleeps until the runner subproces"...

comment "// to respond to status requests"

method_declaration (+126 nodes collapsed)

comment "// initModel sends a load request to the runner based on the"...

comment "// and parameters"

method_declaration (+412 nodes collapsed)

type_declaration

type "type"

type_spec

type_identifier "ServerStatus"

type_identifier "int"

const_declaration (+21 nodes collapsed)

method_declaration (+98 nodes collapsed)

type_declaration (+23 nodes collapsed)

method_declaration (+616 nodes collapsed)

comment "// getServerStatusRetry will retry if ServerStatusNoSlotsAva"...

method_declaration (+141 nodes collapsed)

method_declaration (+79 nodes collapsed)

method_declaration (+685 nodes collapsed)

method_declaration (+63 nodes collapsed)

method_declaration (+26 nodes collapsed)

method_declaration (+71 nodes collapsed)

var_declaration

var "var"

var_spec

identifier "grammarJSON"

= "="

expression_list

raw_string_literal

` "`"

raw_string_literal_content "\nroot ::= object\nvalue ::= object | array | string | numb"...

` "`"

const_declaration

const "const"

const_spec

identifier "maxBufferSize"

= "="

expression_list

binary_expression

int_literal "512"

* "*"

selector_expression

identifier "format"

. "."

field_identifier "KiloByte"

type_declaration (+26 nodes collapsed)

type_declaration (+42 nodes collapsed)

comment "// DoneReason represents the reason why a completion respons"...

type_declaration

type "type"

type_spec

type_identifier "DoneReason"

type_identifier "int"

const_declaration

const "const"

( "("

comment "// DoneReasonStop indicates the completion stopped naturally"

const_spec

identifier "DoneReasonStop"

type_identifier "DoneReason"

= "="

expression_list

iota "iota"

comment "// DoneReasonLength indicates the completion stopped due to "...

const_spec

identifier "DoneReasonLength"

comment "// DoneReasonConnectionClosed indicates the completion stopp"...

const_spec

identifier "DoneReasonConnectionClosed"

) ")"

method_declaration (+59 nodes collapsed)

type_declaration (+64 nodes collapsed)

method_declaration (+1510 nodes collapsed)

type_declaration

type "type"

type_spec

type_identifier "EmbeddingRequest"

struct_type

struct "struct"

field_declaration_list

{ "{"

field_declaration

field_identifier "Content"

type_identifier "string"

raw_string_literal

` "`"

raw_string_literal_content "json:\"content\""

` "`"

} "}"

type_declaration

type "type"

type_spec

type_identifier "EmbeddingResponse"

struct_type

struct "struct"

field_declaration_list

{ "{"

field_declaration

field_identifier "Embedding"

slice_type

[ "["

] "]"

type_identifier "float32"

raw_string_literal

` "`"

raw_string_literal_content "json:\"embedding\""

` "`"

} "}"

method_declaration (+602 nodes collapsed)

type_declaration

type "type"

type_spec

type_identifier "TokenizeRequest"

struct_type

struct "struct"

field_declaration_list

{ "{"

field_declaration

field_identifier "Content"

type_identifier "string"

raw_string_literal

` "`"

raw_string_literal_content "json:\"content\""

` "`"

} "}"

type_declaration

type "type"

type_spec

type_identifier "TokenizeResponse"

struct_type

struct "struct"

field_declaration_list

{ "{"

field_declaration

field_identifier "Tokens"

slice_type

[ "["

] "]"

type_identifier "int"

raw_string_literal

` "`"

raw_string_literal_content "json:\"tokens\""

` "`"

} "}"

method_declaration (+220 nodes collapsed)

type_declaration

type "type"

type_spec

type_identifier "DetokenizeRequest"

struct_type

struct "struct"

field_declaration_list

{ "{"

field_declaration

field_identifier "Tokens"

slice_type

[ "["

] "]"

type_identifier "int"

raw_string_literal

` "`"

raw_string_literal_content "json:\"tokens\""

` "`"

} "}"

type_declaration

type "type"

type_spec

type_identifier "DetokenizeResponse"

struct_type

struct "struct"

field_declaration_list

{ "{"

field_declaration

field_identifier "Content"

type_identifier "string"

raw_string_literal

` "`"

raw_string_literal_content "json:\"content\""

` "`"

} "}"

method_declaration (+245 nodes collapsed)

method_declaration (+234 nodes collapsed)

method_declaration (+29 nodes collapsed)

method_declaration (+94 nodes collapsed)

method_declaration (+48 nodes collapsed)

method_declaration (+191 nodes collapsed)

method_declaration (+104 nodes collapsed)

method_declaration (+90 nodes collapsed)

method_declaration (+110 nodes collapsed)