update runtime options (#864)

2 years ago · 386169205c
1 changed files with 4 additions and 0 deletions
--- a/server/routes.go
+++ b/server/routes.go
@ -112,6 +112,10 @@ func load(ctx context.Context, workDir string, model *Model, reqOpts map[string]
 		loaded.Options = &opts
 	}

+	// update options for the loaded llm
+	// TODO(mxyng): this isn't thread safe, but it should be fine for now
+	loaded.runner.SetOptions(opts)
+
 	loaded.expireAt = time.Now().Add(sessionDuration)

 	if loaded.expireTimer == nil {