Merge branch 'dev' into chore/louis-update-author

janhq · Feb 7, 2024 · eb532ba · eb532ba
2 parents 60d4fd1 31e6bcb
commit eb532ba
Show file tree

Hide file tree

Showing 65 changed files with 1,051 additions and 528 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,39 1,58 @@
-FROM node:20-bullseye AS base
 FROM node:20-bookworm AS base
 
 # 1. Install dependencies only when needed
-FROM base AS deps
 FROM base AS builder
 
 # Install g 11
 RUN apt update && apt install -y gcc-11 g -11 cpp-11 jq xsel && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /app
 
 # Install dependencies based on the preferred package manager
-COPY package.json yarn.lock* package-lock.json* pnpm-lock.yaml* ./
-RUN yarn install
 COPY . ./
 
-# # 2. Rebuild the source code only when needed
-FROM base AS builder
-WORKDIR /app
-COPY --from=deps /app/node_modules ./node_modules
-COPY . .
-# This will do the trick, use the corresponding env file for each environment.
-RUN yarn workspace server install
-RUN yarn server:prod
 RUN export NITRO_VERSION=$(cat extensions/inference-nitro-extension/bin/version.txt) && \
  jq --arg nitroVersion $NITRO_VERSION '(.scripts."downloadnitro:linux" | gsub("\\${NITRO_VERSION}"; $nitroVersion)) | gsub("\r"; "")' extensions/inference-nitro-extension/package.json > /tmp/newcommand.txt && export NEW_COMMAND=$(sed 's/^"//;s/"$//' /tmp/newcommand.txt) && jq --arg newCommand "$NEW_COMMAND" '.scripts."downloadnitro:linux" = $newCommand' extensions/inference-nitro-extension/package.json > /tmp/package.json && mv /tmp/package.json extensions/inference-nitro-extension/package.json
 RUN make install-and-build
 RUN yarn workspace jan-web install
 
 RUN export NODE_ENV=production && yarn workspace jan-web build
 
-# 3. Production image, copy all the files and run next
 # # 2. Rebuild the source code only when needed
 FROM base AS runner
 
 # Install g 11
 RUN apt update && apt install -y gcc-11 g -11 cpp-11 jq xsel && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /app
 
-ENV NODE_ENV=production
 # Copy the package.json and yarn.lock of root yarn space to leverage Docker cache
 COPY --from=builder /app/package.json ./package.json
 COPY --from=builder /app/node_modules ./node_modules/
 COPY --from=builder /app/yarn.lock ./yarn.lock
 
 # Copy the package.json, yarn.lock, and build output of server yarn space to leverage Docker cache
 COPY --from=builder /app/server ./server/
 COPY --from=builder /app/docs/openapi ./docs/openapi/
 
 # Copy pre-install dependencies
 COPY --from=builder /app/pre-install ./pre-install/
 
 # Copy the package.json, yarn.lock, and output of web yarn space to leverage Docker cache
 COPY --from=builder /app/web/out ./web/out/
 COPY --from=builder /app/web/.next ./web/.next/
 COPY --from=builder /app/web/package.json ./web/package.json
 COPY --from=builder /app/web/yarn.lock ./web/yarn.lock
 COPY --from=builder /app/models ./models/
 
-# RUN addgroup -g 1001 -S nodejs;
-COPY --from=builder /app/server/build ./
 RUN npm install -g serve@latest
 
-# Automatically leverage output traces to reduce image size
-# https://nextjs.org/docs/advanced-features/output-file-tracing
-COPY --from=builder /app/server/node_modules ./node_modules
-COPY --from=builder /app/server/package.json ./package.json
 EXPOSE 1337 3000 3928
 
-EXPOSE 4000 3928
 ENV JAN_API_HOST 0.0.0.0
 ENV JAN_API_PORT 1337
 
-ENV PORT 4000
-ENV APPDATA /app/data
 CMD ["sh", "-c", "cd server && node build/main.js & cd web && npx serve out"]
 
-CMD ["node", "main.js"]
 # docker build -t jan .
 # docker run -p 1337:1337 -p 3000:3000 -p 3928:3928 jan
diff --git a/Dockerfile.gpu b/Dockerfile.gpu
@@ -0,0 1,85 @@
 # Please change the base image to the appropriate CUDA version base on NVIDIA Driver Compatibility
 # Run nvidia-smi to check the CUDA version and the corresponding driver version
 # Then update the base image to the appropriate CUDA version refer https://catalog.ngc.nvidia.com/orgs/nvidia/containers/cuda/tags
 
 FROM nvidia/cuda:12.2.0-runtime-ubuntu22.04 AS base 
 
 # 1. Install dependencies only when needed
 FROM base AS builder
 
 # Install g 11
 RUN apt update && apt install -y gcc-11 g -11 cpp-11 jq xsel curl gnupg make python3-dev && curl -sL https://deb.nodesource.com/setup_20.x | bash - && apt install nodejs -y && rm -rf /var/lib/apt/lists/*
 
 # Update alternatives for GCC and related tools
 RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 110 \
  --slave /usr/bin/g g /usr/bin/g -11 \
  --slave /usr/bin/gcov gcov /usr/bin/gcov-11 \
  --slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-11 \
  --slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-11 && \
  update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-11 110
 
 RUN npm install -g yarn
 
 WORKDIR /app
 
 # Install dependencies based on the preferred package manager
 COPY . ./
 
 RUN export NITRO_VERSION=$(cat extensions/inference-nitro-extension/bin/version.txt) && \
  jq --arg nitroVersion $NITRO_VERSION '(.scripts."downloadnitro:linux" | gsub("\\${NITRO_VERSION}"; $nitroVersion)) | gsub("\r"; "")' extensions/inference-nitro-extension/package.json > /tmp/newcommand.txt && export NEW_COMMAND=$(sed 's/^"//;s/"$//' /tmp/newcommand.txt) && jq --arg newCommand "$NEW_COMMAND" '.scripts."downloadnitro:linux" = $newCommand' extensions/inference-nitro-extension/package.json > /tmp/package.json && mv /tmp/package.json extensions/inference-nitro-extension/package.json
 RUN make install-and-build
 RUN yarn workspace jan-web install
 
 RUN export NODE_ENV=production && yarn workspace jan-web build
 
 # # 2. Rebuild the source code only when needed
 FROM base AS runner
 
 # Install g 11
 RUN apt update && apt install -y gcc-11 g -11 cpp-11 jq xsel curl gnupg make python3-dev && curl -sL https://deb.nodesource.com/setup_20.x | bash - && apt-get install nodejs -y && rm -rf /var/lib/apt/lists/*
 
 # Update alternatives for GCC and related tools
 RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 110 \
  --slave /usr/bin/g g /usr/bin/g -11 \
  --slave /usr/bin/gcov gcov /usr/bin/gcov-11 \
  --slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-11 \
  --slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-11 && \
  update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-11 110
 
 RUN npm install -g yarn
 
 WORKDIR /app
 
 # Copy the package.json and yarn.lock of root yarn space to leverage Docker cache
 COPY --from=builder /app/package.json ./package.json
 COPY --from=builder /app/node_modules ./node_modules/
 COPY --from=builder /app/yarn.lock ./yarn.lock
 
 # Copy the package.json, yarn.lock, and build output of server yarn space to leverage Docker cache
 COPY --from=builder /app/server ./server/
 COPY --from=builder /app/docs/openapi ./docs/openapi/
 
 # Copy pre-install dependencies
 COPY --from=builder /app/pre-install ./pre-install/
 
 # Copy the package.json, yarn.lock, and output of web yarn space to leverage Docker cache
 COPY --from=builder /app/web/out ./web/out/
 COPY --from=builder /app/web/.next ./web/.next/
 COPY --from=builder /app/web/package.json ./web/package.json
 COPY --from=builder /app/web/yarn.lock ./web/yarn.lock
 COPY --from=builder /app/models ./models/
 
 RUN npm install -g serve@latest
 
 EXPOSE 1337 3000 3928
 
 ENV LD_LIBRARY_PATH=/usr/local/cuda/targets/x86_64-linux/lib:/usr/local/cuda-12.0/compat${LD_LIBRARY_PATH: :${LD_LIBRARY_PATH}}
 
 ENV JAN_API_HOST 0.0.0.0
 ENV JAN_API_PORT 1337
 
 CMD ["sh", "-c", "cd server && node build/main.js & cd web && npx serve out"]
 
 # pre-requisites: nvidia-docker
 # docker build -t jan-gpu . -f Dockerfile.gpu
 # docker run -p 1337:1337 -p 3000:3000 -p 3928:3928 --gpus all jan-gpu
diff --git a/Makefile b/Makefile
@@ -24,9 24,9 @@ endif
 
 check-file-counts: install-and-build
 ifeq ($(OS),Windows_NT)
- powershell -Command "if ((Get-ChildItem -Path electron/pre-install -Filter *.tgz | Measure-Object | Select-Object -ExpandProperty Count) -ne (Get-ChildItem -Path extensions -Directory | Measure-Object | Select-Object -ExpandProperty Count)) { Write-Host 'Number of .tgz files in electron/pre-install does not match the number of subdirectories in extension'; exit 1 } else { Write-Host 'Extension build successful' }"
  powershell -Command "if ((Get-ChildItem -Path pre-install -Filter *.tgz | Measure-Object | Select-Object -ExpandProperty Count) -ne (Get-ChildItem -Path extensions -Directory | Measure-Object | Select-Object -ExpandProperty Count)) { Write-Host 'Number of .tgz files in pre-install does not match the number of subdirectories in extension'; exit 1 } else { Write-Host 'Extension build successful' }"
 else
- @tgz_count=$$(find electron/pre-install -type f -name "*.tgz" | wc -l); dir_count=$$(find extensions -mindepth 1 -maxdepth 1 -type d | wc -l); if [ $$tgz_count -ne $$dir_count ]; then echo "Number of .tgz files in electron/pre-install ($$tgz_count) does not match the number of subdirectories in extension ($$dir_count)"; exit 1; else echo "Extension build successful"; fi
  @tgz_count=$$(find pre-install -type f -name "*.tgz" | wc -l); dir_count=$$(find extensions -mindepth 1 -maxdepth 1 -type d | wc -l); if [ $$tgz_count -ne $$dir_count ]; then echo "Number of .tgz files in pre-install ($$tgz_count) does not match the number of subdirectories in extension ($$dir_count)"; exit 1; else echo "Extension build successful"; fi
 endif
 
 dev: check-file-counts

diff --git a/README.md b/README.md
@@ -218,6 218,76 @@ make build
 
 This will build the app MacOS m1/m2 for production (with code signing already done) and put the result in `dist` folder.
 
 ### Docker mode
 
 - Supported OS: Linux, WSL2 Docker
 - Pre-requisites:
  - `docker` and `docker compose`, follow instruction [here](https://docs.docker.com/engine/install/ubuntu/)
 
  ```bash
  curl -fsSL https://get.docker.com -o get-docker.sh
  sudo sh ./get-docker.sh --dry-run
  ```
 
  - `nvidia-driver` and `nvidia-docker2`, follow instruction [here](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) (If you want to run with GPU mode)
 
 - Run Jan in Docker mode
 
  - **Option 1**: Run Jan in CPU mode
 
  ```bash
  docker compose --profile cpu up -d
  ```
 
  - **Option 2**: Run Jan in GPU mode
 
  - **Step 1**: Check cuda compatibility with your nvidia driver by running `nvidia-smi` and check the cuda version in the output
 
  ```bash
  nvidia-smi
 
  # Output
  --------------------------------------------------------------------------------------- 
  | NVIDIA-SMI 531.18 Driver Version: 531.18 CUDA Version: 12.1 |
  |----------------------------------------- ---------------------- ---------------------- 
  | GPU Name TCC/WDDM | Bus-Id Disp.A | Volatile Uncorr. ECC |
  | Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |
  | | | MIG M. |
  |========================================= ====================== ======================|
  | 0 NVIDIA GeForce RTX 4070 Ti WDDM | 00000000:01:00.0 On | N/A |
  | 0% 44C P8 16W / 285W| 1481MiB / 12282MiB | 2% Default |
  | | | N/A |
  ----------------------------------------- ---------------------- ---------------------- 
  | 1 NVIDIA GeForce GTX 1660 Ti WDDM | 00000000:02:00.0 Off | N/A |
  | 0% 49C P8 14W / 120W| 0MiB / 6144MiB | 0% Default |
  | | | N/A |
  ----------------------------------------- ---------------------- ---------------------- 
  | 2 NVIDIA GeForce GTX 1660 Ti WDDM | 00000000:05:00.0 Off | N/A |
  | 29% 38C P8 11W / 120W| 0MiB / 6144MiB | 0% Default |
  | | | N/A |
  ----------------------------------------- ---------------------- ---------------------- 
 
  --------------------------------------------------------------------------------------- 
  | Processes: |
  | GPU GI CI PID Type Process name GPU Memory |
  | ID ID Usage |
  |=======================================================================================|
  ```
 
  - **Step 2**: Go to https://catalog.ngc.nvidia.com/orgs/nvidia/containers/cuda/tags and find the smallest minor version of image tag that matches the cuda version from the output of `nvidia-smi` (e.g. 12.1 -> 12.1.0)
 
  - **Step 3**: Update the `Dockerfile.gpu` line number 5 with the latest minor version of the image tag from step 2 (e.g. change `FROM nvidia/cuda:12.2.0-runtime-ubuntu22.04 AS base` to `FROM nvidia/cuda:12.1.0-runtime-ubuntu22.04 AS base`)
 
  - **Step 4**: Run command to start Jan in GPU mode
 
  ```bash
  # GPU mode
  docker compose --profile gpu up -d
  ```
 
  This will start the web server and you can access Jan at `http://localhost:3000`.
  > Note: Currently, Docker mode is only work for development and localhost, production is not supported yet. RAG feature is not supported in Docker mode yet.
 
 ## Acknowledgements
 
 Jan builds on top of other open-source projects:

diff --git a/core/package.json b/core/package.json
@@ -57,6 57,7 @@
  "rollup-plugin-typescript2": "^0.36.0",
  "ts-jest": "^26.1.1",
  "tslib": "^2.6.2",
- "typescript": "^5.2.2"
  "typescript": "^5.2.2",
  "rimraf": "^3.0.2"
  }
 }
diff --git a/core/src/api/index.ts b/core/src/api/index.ts
@@ -30,6 30,7 @@ export enum DownloadRoute {
  downloadFile = 'downloadFile',
  pauseDownload = 'pauseDownload',
  resumeDownload = 'resumeDownload',
  getDownloadProgress = 'getDownloadProgress',
 }
 
 export enum DownloadEvent {

diff --git a/core/src/node/api/routes/download.ts b/core/src/node/api/routes/download.ts
@@ -5,8 5,27 @@ import { HttpServer } from '../HttpServer'
 import { createWriteStream } from 'fs'
 import { getJanDataFolderPath } from '../../utils'
 import { normalizeFilePath } from '../../path'
 import { DownloadState } from '../../../types'
 
 export const downloadRouter = async (app: HttpServer) => {
  app.get(`/${DownloadRoute.getDownloadProgress}/:modelId`, async (req, res) => {
  const modelId = req.params.modelId
 
  console.debug(`Getting download progress for model ${modelId}`)
  console.debug(
  `All Download progress: ${JSON.stringify(DownloadManager.instance.downloadProgressMap)}`
  )
 
  // check if null DownloadManager.instance.downloadProgressMap
  if (!DownloadManager.instance.downloadProgressMap[modelId]) {
  return res.status(404).send({
  message: 'Download progress not found',
  })
  } else {
  return res.status(200).send(DownloadManager.instance.downloadProgressMap[modelId])
  }
  })
 
  app.post(`/${DownloadRoute.downloadFile}`, async (req, res) => {
  const strictSSL = !(req.query.ignoreSSL === 'true')
  const proxy = req.query.proxy?.startsWith('http') ? req.query.proxy : undefined
@@ -19,25 38,55 @@ export const downloadRouter = async (app: HttpServer) => {
  })
 
  const localPath = normalizedArgs[1]
- const fileName = localPath.split('/').pop() ?? ''
  const array = localPath.split('/')
  const fileName = array.pop() ?? ''
  const modelId = array.pop() ?? ''
  console.debug('downloadFile', normalizedArgs, fileName, modelId)
 
  const request = require('request')
  const progress = require('request-progress')
 
  const rq = request({ url: normalizedArgs[0], strictSSL, proxy })
  progress(rq, {})
  .on('progress', function (state: any) {
- console.log('download onProgress', state)
  const downloadProps: DownloadState = {
  ...state,
  modelId,
  fileName,
  downloadState: 'downloading',
  }
  console.debug(`Download ${modelId} onProgress`, downloadProps)
  DownloadManager.instance.downloadProgressMap[modelId] = downloadProps
  })
  .on('error', function (err: Error) {
- console.log('download onError', err)
  console.debug(`Download ${modelId} onError`, err.message)
 
  const currentDownloadState = DownloadManager.instance.downloadProgressMap[modelId]
  if (currentDownloadState) {
  DownloadManager.instance.downloadProgressMap[modelId] = {
  ...currentDownloadState,
  downloadState: 'error',
  }
  }
  })
  .on('end', function () {
- console.log('download onEnd')
  console.debug(`Download ${modelId} onEnd`)
 
  const currentDownloadState = DownloadManager.instance.downloadProgressMap[modelId]
  if (currentDownloadState) {
  if (currentDownloadState.downloadState === 'downloading') {
  // if the previous state is downloading, then set the state to end (success)
  DownloadManager.instance.downloadProgressMap[modelId] = {
  ...currentDownloadState,
  downloadState: 'end',
  }
  }
  }
  })
  .pipe(createWriteStream(normalizedArgs[1]))
 
- DownloadManager.instance.setRequest(fileName, rq)
  DownloadManager.instance.setRequest(localPath, rq)
  res.status(200).send({ message: 'Download started' })
  })
 
  app.post(`/${DownloadRoute.abortDownload}`, async (req, res) => {
@@ -54,5 103,10 @@ export const downloadRouter = async (app: HttpServer) => {
  const rq = DownloadManager.instance.networkRequests[fileName]
  DownloadManager.instance.networkRequests[fileName] = undefined
  rq?.abort()
  if (rq) {
  res.status(200).send({ message: 'Download aborted' })
  } else {
  res.status(404).send({ message: 'Download not found' })
  }
  })
 }