janhq · Van-QA · May 13, 2024 · May 8, 2024 · May 8, 2024 · May 8, 2024
diff --git a/extensions/inference-nitro-extension/package.json b/extensions/inference-nitro-extension/package.json
@@ -1,7 1,7 @@
 {
  "name": "@janhq/inference-nitro-extension",
  "productName": "Nitro Inference Engine",
- "version": "1.0.5",
  "version": "1.0.6",
  "description": "This extension embeds Nitro, a lightweight (3mb) inference engine written in C . See https://nitro.jan.ai.\nAdditional dependencies could be installed to run without Cuda Toolkit installation.",
  "main": "dist/index.js",
  "node": "dist/node/index.cjs.js",

diff --git a/extensions/inference-nitro-extension/resources/models/codeninja-1.0-7b/model.json b/extensions/inference-nitro-extension/resources/models/codeninja-1.0-7b/model.json
@@ -14,7 14,8 @@
  "settings": {
  "ctx_len": 8192,
  "prompt_template": "GPT4 Correct User: {prompt}<|end_of_turn|>GPT4 Correct Assistant:",
- "llama_model_path": "codeninja-1.0-openchat-7b.Q4_K_M.gguf"
  "llama_model_path": "codeninja-1.0-openchat-7b.Q4_K_M.gguf",
  "ngl": 32
  },
  "parameters": {
  "temperature": 0.7,

diff --git a/extensions/inference-nitro-extension/resources/models/command-r-34b/model.json b/extensions/inference-nitro-extension/resources/models/command-r-34b/model.json
@@ -14,7 14,8 @@
  "settings": {
  "ctx_len": 131072,
  "prompt_template": "<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{prompt}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>",
- "llama_model_path": "c4ai-command-r-v01-Q4_K_M.gguf"
  "llama_model_path": "c4ai-command-r-v01-Q4_K_M.gguf",
  "ngl": 40
  },
  "parameters": {
  "temperature": 0.7,

diff --git a/extensions/inference-nitro-extension/resources/models/deepseek-coder-1.3b/model.json b/extensions/inference-nitro-extension/resources/models/deepseek-coder-1.3b/model.json
@@ -14,7 14,8 @@
  "settings": {
  "ctx_len": 16384,
  "prompt_template": "### Instruction:\n{prompt}\n### Response:",
- "llama_model_path": "deepseek-coder-1.3b-instruct.Q8_0.gguf"
  "llama_model_path": "deepseek-coder-1.3b-instruct.Q8_0.gguf",
  "ngl": 24
  },
  "parameters": {
  "temperature": 0.7,

diff --git a/extensions/inference-nitro-extension/resources/models/deepseek-coder-34b/model.json b/extensions/inference-nitro-extension/resources/models/deepseek-coder-34b/model.json
@@ -14,7 14,8 @@
  "settings": {
  "ctx_len": 16384,
  "prompt_template": "### Instruction:\n{prompt}\n### Response:",
- "llama_model_path": "deepseek-coder-33b-instruct.Q4_K_M.gguf"
  "llama_model_path": "deepseek-coder-33b-instruct.Q4_K_M.gguf",
  "ngl": 62
  },
  "parameters": {
  "temperature": 0.7,

diff --git a/extensions/inference-nitro-extension/resources/models/dolphin-phi-2/model.json b/extensions/inference-nitro-extension/resources/models/dolphin-phi-2/model.json
diff --git a/extensions/inference-nitro-extension/resources/models/gemma-2b/model.json b/extensions/inference-nitro-extension/resources/models/gemma-2b/model.json
@@ -14,7 14,8 @@
  "settings": {
  "ctx_len": 8192,
  "prompt_template": "<start_of_turn>user\n{prompt}<end_of_turn>\n<start_of_turn>model",
- "llama_model_path": "gemma-2b-it-q4_k_m.gguf"
  "llama_model_path": "gemma-2b-it-q4_k_m.gguf",
  "ngl": 18
  },
  "parameters": {
  "temperature": 0.7,

diff --git a/extensions/inference-nitro-extension/resources/models/gemma-7b/model.json b/extensions/inference-nitro-extension/resources/models/gemma-7b/model.json
@@ -14,7 14,8 @@
  "settings": {
  "ctx_len": 8192,
  "prompt_template": "<start_of_turn>user\n{prompt}<end_of_turn>\n<start_of_turn>model",
- "llama_model_path": "gemma-7b-it-q4_K_M.gguf"
  "llama_model_path": "gemma-7b-it-q4_K_M.gguf",
  "ngl": 28
  },
  "parameters": {
  "temperature": 0.7,

diff --git a/extensions/inference-nitro-extension/resources/models/llama2-chat-70b/model.json b/extensions/inference-nitro-extension/resources/models/llama2-chat-70b/model.json
@@ -14,7 14,8 @@
  "settings": {
  "ctx_len": 4096,
  "prompt_template": "[INST] <<SYS>>\n{system_message}<</SYS>>\n{prompt}[/INST]",
- "llama_model_path": "llama-2-70b-chat.Q4_K_M.gguf"
  "llama_model_path": "llama-2-70b-chat.Q4_K_M.gguf",
  "ngl": 80
  },
  "parameters": {
  "temperature": 0.7,

diff --git a/extensions/inference-nitro-extension/resources/models/llama2-chat-7b/model.json b/extensions/inference-nitro-extension/resources/models/llama2-chat-7b/model.json
@@ -14,7 14,8 @@
  "settings": {
  "ctx_len": 4096,
  "prompt_template": "[INST] <<SYS>>\n{system_message}<</SYS>>\n{prompt}[/INST]",
- "llama_model_path": "llama-2-7b-chat.Q4_K_M.gguf"
  "llama_model_path": "llama-2-7b-chat.Q4_K_M.gguf",
  "ngl": 32
  },
  "parameters": {
  "temperature": 0.7,

diff --git a/extensions/inference-nitro-extension/resources/models/llama3-8b-instruct/model.json b/extensions/inference-nitro-extension/resources/models/llama3-8b-instruct/model.json
@@ -8,19 8,20 @@
  "id": "llama3-8b-instruct",
  "object": "model",
  "name": "Llama 3 8B Q4",
- "version": "1.0",
  "version": "1.1",
  "description": "Meta's Llama 3 excels at general usage situations, including chat, general world knowledge, and coding.",
  "format": "gguf",
  "settings": {
  "ctx_len": 8192,
  "prompt_template": "<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\n{system_message}<|eot_id|><|start_header_id|>user<|end_header_id|>\n\n{prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",
- "llama_model_path": "Meta-Llama-3-8B-Instruct-Q4_K_M.gguf"
  "llama_model_path": "Meta-Llama-3-8B-Instruct-Q4_K_M.gguf",
  "ngl": 32
  },
  "parameters": {
  "temperature": 0.7,
  "top_p": 0.95,
  "stream": true,
- "max_tokens": 4096,
  "max_tokens": 8192,
  "stop": ["<|end_of_text|>","<|eot_id|>"],
  "frequency_penalty": 0,
  "presence_penalty": 0

diff --git a/...resources/models/hermes-pro-7b/model.json → ...ources/models/llama3-hermes-8b/model.json b/...resources/models/hermes-pro-7b/model.json → ...ources/models/llama3-hermes-8b/model.json
@@ -1,35 1,38 @@
 {
  "sources": [
  {
- "filename": "Hermes-2-Pro-Mistral-7B.Q4_K_M.gguf",
- "url": "https://huggingface.co/NousResearch/Hermes-2-Pro-Mistral-7B-GGUF/resolve/main/Hermes-2-Pro-Mistral-7B.Q4_K_M.gguf"
  "filename": "Hermes-2-Pro-Llama-3-8B-Q4_K_M.gguf",
  "url": "https://huggingface.co/NousResearch/Hermes-2-Pro-Llama-3-8B-GGUF/resolve/main/Hermes-2-Pro-Llama-3-8B-Q4_K_M.gguf"
  }
  ],
- "id": "hermes-pro-7b",
  "id": "llama3-hermes-8b",
  "object": "model",
- "name": "Hermes Pro 7B Q4",
  "name": "Hermes Pro Llama 3 8B Q4",
  "version": "1.1",
- "description": "Hermes Pro is superior in Roleplaying, Reasoning and Explaining problem.",
  "description": "Hermes Pro is well-designed for General chat and JSON output.",
  "format": "gguf",
  "settings": {
- "ctx_len": 4096,
  "ctx_len": 8192,
  "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant",
- "llama_model_path": "Hermes-2-Pro-Mistral-7B.Q4_K_M.gguf"
  "llama_model_path": "Hermes-2-Pro-Llama-3-8B-Q4_K_M.gguf",
  "ngl": 32
  },
  "parameters": {
  "temperature": 0.7,
  "top_p": 0.95,
  "stream": true,
- "max_tokens": 4096,
  "max_tokens": 8192,
  "stop": [],
  "frequency_penalty": 0,
  "presence_penalty": 0
  },
  "metadata": {
  "author": "NousResearch",
- "tags": ["7B", "Finetuned"],
- "size": 4370000000
  "tags": [
  "7B",
  "Finetuned"
  ],
  "size": 4920000000
  },
  "engine": "nitro"
  }
-
diff --git a/extensions/inference-nitro-extension/resources/models/llamacorn-1.1b/model.json b/extensions/inference-nitro-extension/resources/models/llamacorn-1.1b/model.json
@@ -14,7 14,8 @@
  "settings": {
  "ctx_len": 2048,
  "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant",
- "llama_model_path": "llamacorn-1.1b-chat.Q8_0.gguf"
  "llama_model_path": "llamacorn-1.1b-chat.Q8_0.gguf",
  "ngl": 22
  },
  "parameters": {
  "temperature": 0.7,

diff --git a/extensions/inference-nitro-extension/resources/models/miqu-70b/model.json b/extensions/inference-nitro-extension/resources/models/miqu-70b/model.json
diff --git a/extensions/inference-nitro-extension/resources/models/mistral-ins-7b-q4/model.json b/extensions/inference-nitro-extension/resources/models/mistral-ins-7b-q4/model.json
@@ -14,7 14,8 @@
  "settings": {
  "ctx_len": 32768,
  "prompt_template": "[INST] {prompt} [/INST]",
- "llama_model_path": "mistral-7b-instruct-v0.2.Q4_K_M.gguf"
  "llama_model_path": "mistral-7b-instruct-v0.2.Q4_K_M.gguf",
  "ngl": 32
  },
  "parameters": {
  "temperature": 0.7,

diff --git a/extensions/inference-nitro-extension/resources/models/mixtral-8x7b-instruct/model.json b/extensions/inference-nitro-extension/resources/models/mixtral-8x7b-instruct/model.json
@@ -14,7 14,8 @@
  "settings": {
  "ctx_len": 32768,
  "prompt_template": "[INST] {prompt} [/INST]",
- "llama_model_path": "mixtral-8x7b-instruct-v0.1.Q4_K_M.gguf"
  "llama_model_path": "mixtral-8x7b-instruct-v0.1.Q4_K_M.gguf",
  "ngl": 100
  },
  "parameters": {
  "temperature": 0.7,

diff --git a/extensions/inference-nitro-extension/resources/models/noromaid-7b/model.json b/extensions/inference-nitro-extension/resources/models/noromaid-7b/model.json
@@ -14,7 14,8 @@
  "settings": {
  "ctx_len": 32768,
  "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant",
- "llama_model_path": "Noromaid-7B-0.4-DPO.q4_k_m.gguf"
  "llama_model_path": "Noromaid-7B-0.4-DPO.q4_k_m.gguf",
  "ngl": 32
  },
  "parameters": {
  "temperature": 0.7,

diff --git a/extensions/inference-nitro-extension/resources/models/openchat-3.5-7b/model.json b/extensions/inference-nitro-extension/resources/models/openchat-3.5-7b/model.json
@@ -14,7 14,8 @@
  "settings": {
  "ctx_len": 8192,
  "prompt_template": "GPT4 Correct User: {prompt}<|end_of_turn|>GPT4 Correct Assistant:",
- "llama_model_path": "openchat-3.5-0106.Q4_K_M.gguf"
  "llama_model_path": "openchat-3.5-0106.Q4_K_M.gguf",
  "ngl": 32
  },
  "parameters": {
  "temperature": 0.7,

diff --git a/extensions/inference-nitro-extension/resources/models/openhermes-neural-7b/model.json b/extensions/inference-nitro-extension/resources/models/openhermes-neural-7b/model.json
diff --git a/extensions/inference-nitro-extension/resources/models/phi3-3.8b/model.json b/extensions/inference-nitro-extension/resources/models/phi3-3.8b/model.json
diff --git a/extensions/inference-nitro-extension/resources/models/phind-34b/model.json b/extensions/inference-nitro-extension/resources/models/phind-34b/model.json
@@ -14,7 14,8 @@
  "settings": {
  "ctx_len": 16384,
  "prompt_template": "### System Prompt\n{system_message}\n### User Message\n{prompt}\n### Assistant",
- "llama_model_path": "phind-codellama-34b-v2.Q4_K_M.gguf"
  "llama_model_path": "phind-codellama-34b-v2.Q4_K_M.gguf",
  "ngl": 48
  },
  "parameters": {
  "temperature": 0.7,

diff --git a/extensions/inference-nitro-extension/resources/models/qwen-7b/model.json b/extensions/inference-nitro-extension/resources/models/qwen-7b/model.json
@@ -14,7 14,8 @@
  "settings": {
  "ctx_len": 32768,
  "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant",
- "llama_model_path": "qwen1_5-7b-chat-q4_k_m.gguf"
  "llama_model_path": "qwen1_5-7b-chat-q4_k_m.gguf",
  "ngl": 32
  },
  "parameters": {
  "temperature": 0.7,

diff --git a/extensions/inference-nitro-extension/resources/models/stable-zephyr-3b/model.json b/extensions/inference-nitro-extension/resources/models/stable-zephyr-3b/model.json
@@ -14,7 14,8 @@
  "settings": {
  "ctx_len": 4096,
  "prompt_template": "<|user|>\n{prompt}<|endoftext|>\n<|assistant|>",
- "llama_model_path": "stablelm-zephyr-3b.Q8_0.gguf"
  "llama_model_path": "stablelm-zephyr-3b.Q8_0.gguf",
  "ngl": 32
  },
  "parameters": {
  "temperature": 0.7,

diff --git a/extensions/inference-nitro-extension/resources/models/stealth-v1.2-7b/model.json b/extensions/inference-nitro-extension/resources/models/stealth-v1.2-7b/model.json
@@ -14,7 14,8 @@
  "settings": {
  "ctx_len": 32768,
  "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant",
- "llama_model_path": "stealth-v1.3.Q4_K_M.gguf"
  "llama_model_path": "stealth-v1.3.Q4_K_M.gguf",
  "ngl": 32
  },
  "parameters": {
  "temperature": 0.7,

diff --git a/extensions/inference-nitro-extension/resources/models/tinyllama-1.1b/model.json b/extensions/inference-nitro-extension/resources/models/tinyllama-1.1b/model.json
@@ -14,7 14,8 @@
  "settings": {
  "ctx_len": 4096,
  "prompt_template": "<|system|>\n{system_message}<|user|>\n{prompt}<|assistant|>",
- "llama_model_path": "tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf"
  "llama_model_path": "tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf",
  "ngl": 22
  },
  "parameters": {
  "temperature": 0.7,