chore: stop inference event

janhq · hiro-v · Dec 8, 2023 · Dec 1, 2023 · Dec 1, 2023 · Dec 1, 2023
commit b939692187d67ff5d03c5b8f2f6e28412d2330e1
diff --git a/core/src/events.ts b/core/src/events.ts
@@ -18,6 18,8 @@ export enum EventName {
  OnModelStop = "OnModelStop",
  /** The `OnModelStopped` event is emitted when a model stopped ok. */
  OnModelStopped = "OnModelStopped",
  /** The `OnInferenceStopped` event is emitted when a inference is stopped. */
  OnInferenceStopped = "OnInferenceStopped",
 }
 
 /**

diff --git a/core/src/extensions/inference.ts b/core/src/extensions/inference.ts
@@ -5,26 5,10 @@ import { BaseExtension } from "../extension";
  * Inference extension. Start, stop and inference models.
  */
 export abstract class InferenceExtension extends BaseExtension {
- /**
- * Initializes the model for the extension.
- * @param modelId - The ID of the model to initialize.
- */
- abstract initModel(modelId: string, settings?: ModelSettingParams): Promise<void>;
-
- /**
- * Stops the model for the extension.
- */
- abstract stopModel(): Promise<void>;
-
- /**
- * Stops the streaming inference.
- */
- abstract stopInference(): Promise<void>;
-
  /**
  * Processes an inference request.
  * @param data - The data for the inference request.
  * @returns The result of the inference request.
  */
- abstract inferenceRequest(data: MessageRequest): Promise<ThreadMessage>;
  abstract inference(data: MessageRequest): Promise<ThreadMessage>;
 }
diff --git a/extensions/inference-nitro-extension/src/index.ts b/extensions/inference-nitro-extension/src/index.ts
@@ -74,41 74,17 @@ export default class JanInferenceNitroExtension implements InferenceExtension {
  events.on(EventName.OnModelStop, (model: Model) => {
  JanInferenceNitroExtension.handleModelStop(model);
  });
- }
 
- /**
- * Stops the model inference.
- */
- onUnload(): void {
- this.stopModel();
  events.on(EventName.OnInferenceStopped, () => {
  JanInferenceNitroExtension.handleInferenceStopped(this);
  });
  }
 
  /**
- * Initializes the model with the specified file name.
- * @param {string} modelId - The ID of the model to initialize.
- * @returns {Promise<void>} A promise that resolves when the model is initialized.
  * Stops the model inference.
  */
- async initModel(
- modelId: string,
- settings?: ModelSettingParams
- ): Promise<void> {}
  onUnload(): void {}
 
- /**
- * Stops the model.
- * @returns {Promise<void>} A promise that resolves when the model is stopped.
- */
- async stopModel(): Promise<void> {
- return executeOnMain(MODULE, "killSubprocess");
- }
-
- /**
- * Stops streaming inference.
- * @returns {Promise<void>} A promise that resolves when the streaming is stopped.
- */
- async stopInference(): Promise<void> {
- this.isCancelled = true;
- this.controller?.abort();
- }
 
  private async writeDefaultEngineSettings() {
  try {
@@ -160,12 136,19 @@ export default class JanInferenceNitroExtension implements InferenceExtension {
  }
  }
 
  private static async handleInferenceStopped(
  instance: JanInferenceNitroExtension
  ) {
  instance.isCancelled = true;
  instance.controller?.abort();
  }
 
  /**
  * Makes a single response inference request.
  * @param {MessageRequest} data - The data for the inference request.
  * @returns {Promise<any>} A promise that resolves with the inference response.
  */
- async inferenceRequest(data: MessageRequest): Promise<ThreadMessage> {
  async inference(data: MessageRequest): Promise<ThreadMessage> {
  const timestamp = Date.now();
  const message: ThreadMessage = {
  thread_id: data.threadId,

diff --git a/extensions/inference-openai-extension/src/index.ts b/extensions/inference-openai-extension/src/index.ts
@@ -71,25 71,16 @@ export default class JanInferenceOpenAIExtension implements InferenceExtension {
  events.on(EventName.OnModelStop, (model: OpenAIModel) => {
  JanInferenceOpenAIExtension.handleModelStop(model);
  });
  events.on(EventName.OnInferenceStopped, () => {
  JanInferenceOpenAIExtension.handleInferenceStopped(this);
  });
  }
 
  /**
  * Stops the model inference.
  */
  onUnload(): void {}
 
- /**
- * Initializes the model with the specified file name.
- * @param {string} modelId - The ID of the model to initialize.
- * @returns {Promise<void>} A promise that resolves when the model is initialized.
- */
- async initModel(
- modelId: string,
- settings?: ModelSettingParams
- ): Promise<void> {
- return;
- }
-
  static async writeDefaultEngineSettings() {
  try {
  const engineFile = join(
@@ -110,27 101,13 @@ export default class JanInferenceOpenAIExtension implements InferenceExtension {
  console.error(err);
  }
  }
- /**
- * Stops the model.
- * @returns {Promise<void>} A promise that resolves when the model is stopped.
- */
- async stopModel(): Promise<void> {}
-
- /**
- * Stops streaming inference.
- * @returns {Promise<void>} A promise that resolves when the streaming is stopped.
- */
- async stopInference(): Promise<void> {
- this.isCancelled = true;
- this.controller?.abort();
- }
 
  /**
  * Makes a single response inference request.
  * @param {MessageRequest} data - The data for the inference request.
  * @returns {Promise<any>} A promise that resolves with the inference response.
  */
- async inferenceRequest(data: MessageRequest): Promise<ThreadMessage> {
  async inference(data: MessageRequest): Promise<ThreadMessage> {
  const timestamp = Date.now();
  const message: ThreadMessage = {
  thread_id: data.threadId,
@@ -168,7 145,6 @@ export default class JanInferenceOpenAIExtension implements InferenceExtension {
  JanInferenceOpenAIExtension.writeDefaultEngineSettings();
  // Todo: Check model list with API key
  events.emit(EventName.OnModelReady, model);
- // events.emit(EventName.OnModelFail, model)
  }
  }
 
@@ -179,6 155,13 @@ export default class JanInferenceOpenAIExtension implements InferenceExtension {
  events.emit(EventName.OnModelStopped, model);
  }
 
  private static async handleInferenceStopped(
  instance: JanInferenceOpenAIExtension
  ) {
  instance.isCancelled = true;
  instance.controller?.abort();
  }
 
  /**
  * Handles a new message request by making an inference request and emitting events.
  * Function registered in event manager, should be static to avoid binding issues.

diff --git a/web/screens/Chat/MessageToolbar/index.tsx b/web/screens/Chat/MessageToolbar/index.tsx
@@ -30,9 30,8 @@ const MessageToolbar = ({ message }: { message: ThreadMessage }) => {
  const { resendChatMessage } = useSendChatMessage()
 
  const onStopInferenceClick = async () => {
- await extensionManager
- .get<InferenceExtension>(ExtensionType.Inference)
- ?.stopInference()
  events.emit(EventName.OnInferenceStopped, {})
 
  setTimeout(() => {
  events.emit(EventName.OnMessageUpdate, {
  ...message,

diff --git a/web/screens/ExploreModels/ExploreModelItemHeader/index.tsx b/web/screens/ExploreModels/ExploreModelItemHeader/index.tsx
@@ -55,23 55,9 @@ const ExploreModelItemHeader: React.FC<Props> = ({ model, onClick, open }) => {
 
  const isDownloaded = downloadedModels.find((md) => md.id === model.id) != null
 
- let downloadButton;
-
- if (model.engine === 'openai') {
- downloadButton = (
- <Button onClick={() => onDownloadClick()}>
- Use
- </Button>
- );
- } else if (model.engine === 'nitro') {
- downloadButton = (
- <Button onClick={() => onDownloadClick()}>
- {model.metadata.size
- ? `Download (${toGigabytes(model.metadata.size)})`
- : 'Download'}
- </Button>
- );
- }
  let downloadButton = (
  <Button onClick={() => onDownloadClick()}>Download</Button>
  )
 
  const onUseModelClick = () => {
  startModel(model.id)