Merge pull request #25 from stv0g/fix-23

Fix MinIO setup and add Docker Compose file
stv0g · Jun 10, 2022 · 8d5c2f1 · 8d5c2f1
2 parents e1ddcd4 832ee56
commit 8d5c2f1
Show file tree

Hide file tree

Showing 16 changed files with 330 additions and 100 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -27,7  27,7 @@ RUN npm run build
 
 FROM alpine:3.15
 
-RUN apk update && apk add ca-certificates && rm -rf /var/cache/apk/*
 RUN apk update && apk add ca-certificates curl && rm -rf /var/cache/apk/*
 
 COPY --from=frontend-builder /app/dist/ /dist/
 COPY --from=backend-builder /app/gose /
@@ -36,4  36,9 @@ COPY --from=backend-builder /app/config.yaml /
 ENV GIN_MODE=release
 ENV GOSE_SERVER_STATIC=/dist
 
 EXPOSE 8080/tcp
 
 HEALTHCHECK --interval=30s --timeout=30s --retries=3 \
     CMD curl -f http://localhost:8080/api/v1/healthz
 
 ENTRYPOINT [ "/gose" ]
diff --git a/Dockerfile.release b/Dockerfile.release
@@ -1,9  1,14 @@
 FROM alpine:3.15
 
-RUN apk update && apk add ca-certificates && rm -rf /var/cache/apk/*
 RUN apk update && apk add ca-certificates curl && rm -rf /var/cache/apk/*
 
 ENV GIN_MODE=release
 
 EXPOSE 8080/tcp
 
 HEALTHCHECK --interval=30s --timeout=30s --retries=3 \
     CMD curl -f http://localhost:8080/api/v1/healthz
 
 COPY gose /
 
 ENTRYPOINT ["/gose"]
diff --git a/README.md b/README.md
@@ -111,30  111,25 @@ For reference have a look at the [example configuration file](config.yaml).
 
 All settings from the configuration file can also be set via environment variables:
 
-| Variable                              | Example Value                                                             | Description                           |
-| :--                                   | :--                                                                       | :--                                   |
-| `GOSE_LISTEN`                         | `":8080"`                                                                 | Listen address and port of Gose       |
-| `GOSE_BASE_URL`                       | `"http://localhost:8080"`                                                 | Base URL at which Gose is accessible  |
-| `GOSE_STATIC`                         | `"./dist"`                                                                | Directory of frontend assets if not bundled into the binary |
-| `GOSE_BUCKET`                         | `gose-uploads`                                                            | Name of S3 bucket                     |
-| `GOSE_ENDPOINT`                       | `s3.0l.de`                                                                | Hostname of S3 server                 |
-| `GOSE_REGION`                         | `s3`                                                                      | Region of S3 server                   |
-| `GOSE_PATH_STYLE`                     | `true`                                                                    | Prepend bucket name to path           |
-| `GOSE_NO_SSL`                         | `false`                                                                   | Disable SSL encryption for S3         |
-| `GOSE_ACCESS_KEY`                     |                                                                           | S3 Access Key                         |
-| `GOSE_SECRET_KEY`                     |                                                                           | S3 Secret Key                         |
-| `AWS_ACCESS_KEY_ID`                   |                                                                           | alias for `GOSE_S3_ACCESS_KEY`        |
-| `AWS_SECRET_ACCESS_KEY`               |                                                                           | alias for `AWS_SECRET_ACCESS_KEY`     |
-| `GOSE_S3_MAX_UPLOAD_SIZE`             | `5TB`                                                                     | Maximum upload size                   |
-| `GOSE_S3_PART_SIZE`                   | `5MB`                                                                     | Part-size for multi-part uploads      |
-| `GOSE_S3_EXPIRATION_DEFAULT_CLASS`    | `1week # one of the tags below`                                           | Default expiration class              |
-| `GOSE_SHORTENER_ENDPOINT`             | `"https://shlink-api/rest/v2/short-urls/shorten?apiKey=<your-api-token>&format=txt&longUrl={{.UrlEscaped}}"`  | API Endpoint of link shortener |
-| `GOSE_SHORTENER_METHOD`               | `GET`                                                                     | HTTP method for link shortener        |
-| `GOSE_SHORTENER_RESPONSE`             | `raw`                                                                     | Response type of link shortener       |
-| `GOSE_NOTIFICATION_URLS`              | `pushover://shoutrrr:<api-token>@<user-key>?devices=laptop1&title=Upload` | Service URLs for [shoutrrr notifications](https://containrrr.dev/shoutrrr/) |
-| `GOSE_NOTIFICATION_TEMPLATE`          | `"New Upload: {{.URL}}"`                                                  | Notification message template         |
-| `GOSE_NOTIFICATION_MAIL_URL`          | `smtp://user:password@host:port/[email protected]`             | Service URLs for [shoutrrr notifications](https://containrrr.dev/shoutrrr/) |
-| `GOSE_NOTIFICATION_MAIL_TEMPLATE`     | `"New Upload: {{.URL}}"`                                                  | Notification message template         |
 | Variable                               | Default Value                                                             | Description                           |
 | :--                                    | :--                                                                       | :--                                   |
 | `GOSE_LISTEN`                          | `":8080"`                                                                 | Listen address and port of Gose       |
 | `GOSE_BASE_URL`                        | `"http://localhost:8080"`                                                 | Base URL at which Gose is accessible  |
 | `GOSE_STATIC`                          | `"./dist"`                                                                | Directory of frontend assets if not bundled into the binary |
 | `GOSE_BUCKET`                          | `gose-uploads`                                                            | Name of S3 bucket                     |
 | `GOSE_ENDPOINT`                        | (without `http(s)://` prefix, but with port number)                       | Hostname:Port of S3 server            |
 | `GOSE_REGION`                          | `us-east-1`                                                               | Region of S3 server                   |
 | `GOSE_PATH_STYLE`                      | `false`                                                                   | Prepend bucket name to path           |
 | `GOSE_NO_SSL`                          | `false`                                                                   | Disable SSL encryption for S3         |
 | `GOSE_ACCESS_KEY`                      |                                                                           | S3 Access Key                         |
 | `GOSE_SECRET_KEY`                      |                                                                           | S3 Secret Key                         |
 | `GOSE_CREATE_BUCKET`                   | `true`                                                                    | Create S3 bucket if non-existant      |
 | `GOSE_MAX_UPLOAD_SIZE`                 | `1TB`                                                                     | Maximum upload size                   |
 | `GOSE_PART_SIZE`                       | `16MB`                                                                    | Part-size for multi-part uploads      |
 | `AWS_ACCESS_KEY_ID`                    |                                                                           | alias for `GOSE_ACCESS_KEY`           |
 | `AWS_SECRET_ACCESS_KEY`                |                                                                           | alias for `GOSE_SECRET_KEY`           |
 
 Configuration of link shortener and notifiers must be done via a [configuration file](#file).
 
 ## Author
 

diff --git a/cmd/main.go b/cmd/main.go
@@ -26,6  26,8 @@ var (
 const apiBase = "/api/v1"
 
 func main() {
 	log.Printf("GoSƐ %s, commit %s, built at %s by %s", version, commit, date, builtBy)
 
 	// Generate our config based on the config supplied
 	// by the user in the flags
 	cfgFile, err := config.ParseFlags()
@@ -53,14  55,15 @@ func APIMiddleware(svrs server.List, shortener *shortener.Shortener, cfg *config
 }
 
 func run(cfg *config.Config) {
-	var err error
-
 	svrs := server.NewList(cfg.Servers)
 
 	log.Printf("Initializing S3 servers. Please wait...")
 	if err := svrs.Setup(); err != nil {
 		log.Fatalf("Failed to setup servers: %s", err)
 	}
 	log.Printf("Initialization of %d servers completed.", len(svrs))
 
 	var err error
 	var short *shortener.Shortener
 	if cfg.Shortener != nil {
 		if short, err = shortener.NewShortener(cfg.Shortener); err != nil {
@@ -73,6  76,7 @@ func run(cfg *config.Config) {
 	router.Use(StaticMiddleware(cfg))
 
 	router.GET(apiBase "/config", handlers.HandleConfigWith(version, commit, date))
 	router.GET(apiBase "/healthz", handlers.HandleHealthz)
 	router.POST(apiBase "/initiate", handlers.HandleInitiate)
 	router.POST(apiBase "/part", handlers.HandlePart)
 	router.POST(apiBase "/complete", handlers.HandleComplete)
@@ -87,7  91,6 @@ func run(cfg *config.Config) {
 		MaxHeaderBytes: 1 << 20,
 	}
 
-	log.Printf("GoSƐ %s, commit %s, built at %s by %s", version, commit, date, builtBy)
 	log.Printf("Listening on: http://%s", server.Addr)
 
 	server.ListenAndServe()

diff --git a/config.yaml b/config.yaml
@@ -16,11  16,11 @@ max_upload_size: 1TB
 servers:
 - bucket: gose-uploads
 
-  endpoint: s3.0l.de
   endpoint: localhost:9500
   region: s3
 
   path_style: true
-  no_ssl: false
   no_ssl: true
 
   # Or via standard AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY env vars
   access_key: ""

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -0,0  1,46 @@
 ---
 version: "3.7" 
 services:
 
   minio:
     image: minio/minio:RELEASE.2022-06-03T01-40-53Z.fips
     command: server /mnt/data --console-address ":9001"
     ports:
     - 9500:9500 # API
     - 9001:9001 # Webinterface
     environment:
       MINIO_ROOT_USER: "admin-user" # changeme!
       MINIO_ROOT_PASSWORD: "admin-pass"  # changeme!
       MINIO_SERVER_URL: "http://localhost:9500"
       MINIO_SITE_REGION: "s3"
     volumes:
     - minio-data:/mnt/data
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:9500/minio/health/ready"]
       interval: 30s
       timeout: 20s
       retries: 3
 
   gose:
     image: ghcr.io/stv0g/gose:v0.4.0
     build:
       context: .
     ports:
     - 8080:8080
     environment:
       GOSE_LISTEN: ":8080"
       GOSE_BASE_URL: "http://localhost:8080"
       GOSE_BUCKET: "gose-uploads"
       GOSE_ENDPOINT: "minio:9500"
       GOSE_REGION: "s3"
       GOSE_PATH_STYLE: "true"
       GOSE_NO_SSL: "true"
       GOSE_ACCESS_KEY: "admin-user" # changeme!
       GOSE_SECRET_KEY: "admin-pass" # changeme!
       GOSE_MAX_UPLOAD_SIZE: "50GB"
       GOSE_PART_SIZE: "16MB"
     depends_on:
     - minio
 
 volumes:
   minio-data:
diff --git a/frontend/index.html b/frontend/index.html
@@ -20,7  20,7 @@ <h4 class="subtitle">A terascale file uploader</h4>
         <div class="container">
             <div class="row">
                 <div class="col-12 gy-3">
-                    <div id="result" class="alert mb-0 align-items-center d-flex d-none" role="alert"></div>
                     <div id="result" class="alert mb-0 align-items-center justify-content-between d-flex d-none" role="alert"></div>
                 </div>
                 <div class="col-12 gy-3">
                     <div id="statistics" class="card d-none">

diff --git a/frontend/src/index.ts b/frontend/src/index.ts
@@ -15,11  15,13 @@ import { apiRequest } from "./api";
 import { Config, Server } from "./config";
 import { Chart } from "./chart";
 import { Dropzone } from "./dropzone";
 import { shortUrl } from "./utils";
 
 var progressBar: ProgressBar;
 var config: Config;
 var chart: Chart;
 var upload: Upload | null;
 var servers: Record<string,Server> = {}
 let points: Array<number[]> = [];
 
 function reset() {
@@ -64,16  66,17 @@ function alert(cls: string, msg: string, url?: string, icon?: string) {
     elm.innerHTML  = `<span>${msg}</span>`;
 
     if (url) {
-        elm.innerHTML  = `<a class="alert-link ms-auto" id="copy" data-bs-toggle="tooltip" data-bs-placement="top" title="Copy to clipboard"><i class="fa-solid fa-copy"></i></a>`;
-        elm.innerHTML  = `<a class="alert-link" id="upload-url" href="${url}">${url}</a>`;
         let displayUrl = shortUrl(url);
         elm.innerHTML  = `<a class="alert-link" id="upload-url" href="${url}">${displayUrl}</a>`;
         elm.innerHTML  = `<a class="alert-link" id="copy" data-bs-toggle="tooltip" data-bs-placement="top" title="Copy to clipboard"><i class="fa-solid fa-copy"></i></a>`;
 
         // Setup copy to clipboard
         let btnCopy = document.getElementById("copy");
         let spanUrl = document.getElementById("upload-url");
         let tooltip = new Tooltip(btnCopy);
 
         btnCopy.addEventListener("click", async (ev: Event) => {
-            await navigator.clipboard.writeText(spanUrl.innerText);
             await navigator.clipboard.writeText(url);
 
             tooltip.dispose();
             btnCopy.title = "Copied! 🥳";
@@ -172,7  175,7 @@ async function startUpload(files: FileList) {
             start: uploadStarted,
             end: uploadEnded,
             progress: uploadProgressed,
-        }, params);
         }, params, servers[params.server]);
 
         let url = await upload.start();
 
@@ -296,6  299,8 @@ function onConfig(config: Config) {
         opt.value = svr.id;
         opt.innerHTML = svr.title;
         selServers.appendChild(opt);
 
         servers[svr.id] = svr;
     }
 
     if (config.servers.length > 1) {

diff --git a/frontend/src/upload.ts b/frontend/src/upload.ts
@@ -2,6  2,7 @@ import { ProgressHandler } from "./progress-handler";
 import { buf2hex, hex2buf, arraybufferEqual } from "./utils";
 import { apiRequest} from "./api";
 import * as md5 from "js-md5";
 import { Config, Server } from "./config";
 
 export class UploadParams {
     server: string
@@ -51,17  52,16 @@ export class Upload {
     protected parts: Part[] = [];
     protected callbacks: Callbacks;
     protected params: UploadParams;
     protected server: Server;
     protected xhr: XMLHttpRequest;
 
-    // TODO: take this from the configuration
-    readonly partSize = 6e6;
-
-    constructor(file: File, cbs: Callbacks, params: UploadParams) {
     constructor(file: File, cbs: Callbacks, params: UploadParams, server: Server) {
         this.file = file;
         this.callbacks = cbs;
         this.params = params;
         this.server = server;
 
-        const partsCount = Math.ceil(this.file.size / this.partSize);
         const partsCount = Math.ceil(this.file.size / this.server.part_size);
 
         this.progress = new ProgressHandler({
             start: () => this.callbacks.start(this),
@@ -95,8  95,8 @@ export class Upload {
 
         let parts: Part[] = [];
         let partNumber = 1;
-        for (let offset = 0; offset < this.file.size; offset  = this.partSize) {
-            let length = this.partSize;
         for (let offset = 0; offset < this.file.size; offset  = this.server.part_size) {
             let length = this.server.part_size;
             if (offset   length > this.file.size) {
                 length = this.file.size - offset; // handle last part
             }
@@ -114,7  114,7 @@ export class Upload {
 
             let md = md5.create();
             // let chunkSize = 1<<20;
-            let chunkSize = this.partSize;
             let chunkSize = this.server.part_size;
             for (let chunkOffset = 0; chunkOffset < partBuffer.byteLength; chunkOffset  = chunkSize) {
                 let chunkLength = chunkSize;
                 if (chunkOffset   chunkSize > partBuffer.byteLength) {

diff --git a/frontend/src/utils.ts b/frontend/src/utils.ts
@@ -29,3  29,38 @@ export function arraybufferEqual(a: ArrayBuffer, b: ArrayBuffer) {
 
     return true;
 }
 
 export function shortUrl(url: string, l?: number){
     var l = typeof(l) != "undefined" ? l : 32;
     var chunk_l = (l/2);
     var url = url.replace("http://","").replace("https://","");
 
     if (url.length <= l) {
         return url;
     }
 
     var start_chunk = shortString(url, chunk_l, false);
     var end_chunk = shortString(url, chunk_l, true);
     return start_chunk   ".."   end_chunk;
 }
 
 function shortString(s: string, l: number, reverse?: boolean){
     var stop_chars = [' ','/', '&'];
     var acceptable_shortness = l * 0.80; // When to start looking for stop characters
     var reverse = typeof(reverse) != "undefined" ? reverse : false;
     var s = reverse ? s.split("").reverse().join("") : s;
     var short_s = "";
 
     for (var i=0; i < l-1; i  ) {
         short_s  = s[i];
         if (i >= acceptable_shortness && stop_chars.indexOf(s[i]) >= 0) {
             break;
         }
     }
 
     if (reverse) {
         return short_s.split("").reverse().join("");
     }
 
     return short_s;
 }