Enable num_gpu layers to be edited and used by ollama

derpyhue · Jul 4, 2024 · fff91f7 · fff91f7
1 parent 824966a
commit fff91f7
Show file tree

Hide file tree

Showing 4 changed files with 52 additions and 0 deletions.
diff --git a/backend/apps/ollama/main.py b/backend/apps/ollama/main.py
@@ -759,6 759,9 @@ async def generate_chat_completion(
 
  if model_info.params.get("num_ctx", None):
  payload["options"]["num_ctx"] = model_info.params.get("num_ctx", None)
 
  if model_info.params.get("num_gpu", None):
  payload["options"]["num_gpu"] = model_info.params.get("num_gpu", None)
 
  if model_info.params.get("num_batch", None):
  payload["options"]["num_batch"] = model_info.params.get(

diff --git a/src/lib/components/chat/Settings/Advanced/AdvancedParams.svelte b/src/lib/components/chat/Settings/Advanced/AdvancedParams.svelte
@@ -21,6 21,7 @@
  top_p: null,
  tfs_z: null,
  num_ctx: null,
  num_gpu: null,
  num_batch: null,
  num_keep: null,
  max_tokens: null,
@@ -566,6 567,51 @@
  </div>
  {/if}
  </div>
 
  <div class=" py-0.5 w-full justify-between">
  <div class="flex w-full justify-between">
  <div class=" self-center text-xs font-medium">{$i18n.t('GPU Layers')}</div>
 
  <button
  class="p-1 px-3 text-xs flex rounded transition"
  type="button"
  on:click={() => {
  params.num_gpu = (params?.num_gpu ?? null) === null ? 100 : null;
  }}
  >
  {#if (params?.num_gpu ?? null) === null}
  <span class="ml-2 self-center">{$i18n.t('Default')}</span>
  {:else}
  <span class="ml-2 self-center">{$i18n.t('Custom')}</span>
  {/if}
  </button>
  </div>
 
  {#if (params?.num_gpu ?? null) !== null}
  <div class="flex mt-0.5 space-x-2">
  <div class=" flex-1">
  <input
  id="steps-range"
  type="range"
  min="-1"
  max="999"
  step="1"
  bind:value={params.num_gpu}
  class="w-full h-2 rounded-lg appearance-none cursor-pointer dark:bg-gray-700"
  />
  </div>
  <div class="">
  <input
  bind:value={params.num_gpu}
  type="number"
  class=" bg-transparent text-center w-14"
  min="-1"
  step="1"
  />
  </div>
  </div>
  {/if}
  </div>
 
  <div class=" py-0.5 w-full justify-between">
  <div class="flex w-full justify-between">

diff --git a/src/lib/components/chat/Settings/General.svelte b/src/lib/components/chat/Settings/General.svelte
@@ -57,6 57,7 @@
  stop: null,
  tfs_z: null,
  num_ctx: null,
  num_gpu: null,
  num_batch: null,
  num_keep: null,
  max_tokens: null
@@ -312,6 313,7 @@
  top_p: params.top_p !== null ? params.top_p : undefined,
  tfs_z: params.tfs_z !== null ? params.tfs_z : undefined,
  num_ctx: params.num_ctx !== null ? params.num_ctx : undefined,
  num_gpu: params.num_gpu !== null ? params.num_gpu : undefined,
  num_batch: params.num_batch !== null ? params.num_batch : undefined,
  num_keep: params.num_keep !== null ? params.num_keep : undefined,
  max_tokens: params.max_tokens !== null ? params.max_tokens : undefined,

diff --git a/src/lib/stores/index.ts b/src/lib/stores/index.ts
@@ -94,6 94,7 @@ type Settings = {
  top_k?: string;
  top_p?: string;
  num_ctx?: string;
  num_gpu?: string;
  num_batch?: string;
  num_keep?: string;
  options?: ModelOptions;