ajout app

This commit is contained in:
2024-04-17 20:22:30 +02:00
parent cc017cfc5e
commit f9d05a2fd3
8025 changed files with 729805 additions and 0 deletions

View File

@ -0,0 +1,44 @@
root = "."
testdata_dir = "testdata"
tmp_dir = ".air"
[build]
args_bin = []
bin = "./.air/main"
cmd = "go build -o ./.air/main ."
delay = 0
exclude_dir = ["assets", "tmp", "vendor", "testdata"]
exclude_file = []
exclude_regex = ["_test.go"]
exclude_unchanged = false
follow_symlink = false
full_bin = ""
include_dir = []
include_ext = ["go", "tpl", "tmpl", "html"]
include_file = []
kill_delay = "0s"
log = "build-errors.log"
poll = false
poll_interval = 0
rerun = false
rerun_delay = 500
send_interrupt = false
stop_on_error = false
[color]
app = ""
build = "yellow"
main = "magenta"
runner = "green"
watcher = "cyan"
[log]
main_only = false
time = false
[misc]
clean_on_exit = false
[screen]
clear_on_rebuild = false
keep_scroll = true

View File

@ -0,0 +1,22 @@
PORT=5001
# URLs
API_URL="http://127.0.0.1:5000"
# Security
SECURITY_API_KEY="7znl9Zd8F!4lRZA43lEQb757mCy"
# S3
S3_URL="127.0.0.1:9000"
S3_ACCESS_KEY="voltaserve"
S3_SECRET_KEY="voltaserve"
S3_REGION="us-east-1"
S3_SECURE=false
# Limits
LIMITS_EXTERNAL_COMMAND_TIMEOUT_SECONDS=900
LIMITS_FILE_PROCESSING_MAX_SIZE_MB=100
LIMITS_IMAGE_PREVIEW_MAX_WIDTH=512
LIMITS_IMAGE_PREVIEW_MAX_HEIGHT=512
LIMITS_MULTIPART_BODY_LENGTH_LIMIT_MB=1000
LIMITS_LANGUAGE_SCORE_THRESHOLD=0.97

5
Voltaserve/conversion/.gitignore vendored Normal file
View File

@ -0,0 +1,5 @@
/.env.local
/voltaserve
__debug_bin*
/*.tsv
/.air

View File

@ -0,0 +1,5 @@
{
"recommendations": [
"golang.go"
]
}

View File

@ -0,0 +1,15 @@
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"name": "Launch Package",
"type": "go",
"request": "launch",
"mode": "auto",
"program": "main.go"
}
]
}

View File

@ -0,0 +1,6 @@
{
"[go]": {
"editor.defaultFormatter": "golang.go",
"editor.tabSize": 4
}
}

View File

@ -0,0 +1,24 @@
FROM registry.suse.com/bci/golang:1.21 AS builder
WORKDIR /build
COPY . .
RUN go mod download
RUN go build -o voltaserve-conversion
FROM opensuse/leap:15.5 AS runner
COPY --from=builder /build/scripts ./scripts
RUN ./scripts/install-dependencies.sh
RUN ./scripts/install-fonts.sh
WORKDIR /app
COPY --from=builder /build/voltaserve-conversion ./voltaserve-conversion
COPY --from=builder /build/.env ./.env
ENTRYPOINT ["./voltaserve-conversion"]
EXPOSE 5001

View File

@ -0,0 +1,27 @@
# Voltaserve Conversion
Install [golangci-lint](https://golangci-lint.run/usage/install).
Run for development:
```shell
go run .
```
Build binary:
```shell
go build .
```
Run code linter:
```shell
golangci-lint run
```
Build Docker image:
```shell
docker build -t voltaserve/conversion .
```

View File

@ -0,0 +1,49 @@
package client
import (
"bytes"
"encoding/json"
"fmt"
"go.uber.org/zap"
"net/http"
"voltaserve/config"
"voltaserve/core"
"voltaserve/infra"
)
type APIClient struct {
config config.Config
logger *zap.SugaredLogger
}
func NewAPIClient() *APIClient {
logger, err := infra.GetLogger()
if err != nil {
panic(err)
}
return &APIClient{
config: config.GetConfig(),
logger: logger,
}
}
func (cl *APIClient) UpdateSnapshot(opts core.SnapshotUpdateOptions) error {
body, err := json.Marshal(opts)
if err != nil {
return err
}
req, err := http.NewRequest("POST", fmt.Sprintf("%s/v1/files/conversion_webhook/update_snapshot?api_key=%s", cl.config.APIURL, cl.config.Security.APIKey), bytes.NewBuffer(body))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json; charset=UTF-8")
client := &http.Client{}
res, err := client.Do(req)
if err != nil {
return err
}
if err := res.Body.Close(); err != nil {
return err
}
return nil
}

View File

@ -0,0 +1,85 @@
package config
import (
"os"
"strconv"
)
var config *Config
func GetConfig() Config {
if config == nil {
port, err := strconv.Atoi(os.Getenv("PORT"))
if err != nil {
panic(err)
}
config = &Config{
Port: port,
}
readURLs(config)
readSecurity(config)
readS3(config)
readLimits(config)
}
return *config
}
func readURLs(config *Config) {
config.APIURL = os.Getenv("API_URL")
}
func readSecurity(config *Config) {
config.Security.APIKey = os.Getenv("SECURITY_API_KEY")
}
func readS3(config *Config) {
config.S3.URL = os.Getenv("S3_URL")
config.S3.AccessKey = os.Getenv("S3_ACCESS_KEY")
config.S3.SecretKey = os.Getenv("S3_SECRET_KEY")
config.S3.Region = os.Getenv("S3_REGION")
if len(os.Getenv("S3_SECURE")) > 0 {
v, err := strconv.ParseBool(os.Getenv("S3_SECURE"))
if err != nil {
panic(err)
}
config.S3.Secure = v
}
}
func readLimits(config *Config) {
if len(os.Getenv("LIMITS_EXTERNAL_COMMAND_TIMEOUT_SECONDS")) > 0 {
v, err := strconv.ParseInt(os.Getenv("LIMITS_EXTERNAL_COMMAND_TIMEOUT_SECONDS"), 10, 32)
if err != nil {
panic(err)
}
config.Limits.ExternalCommandTimeoutSeconds = int(v)
}
if len(os.Getenv("LIMITS_FILE_PROCESSING_MAX_SIZE_MB")) > 0 {
v, err := strconv.ParseInt(os.Getenv("LIMITS_FILE_PROCESSING_MAX_SIZE_MB"), 10, 32)
if err != nil {
panic(err)
}
config.Limits.FileProcessingMaxSizeMB = int(v)
}
if len(os.Getenv("LIMITS_IMAGE_PREVIEW_MAX_WIDTH")) > 0 {
v, err := strconv.ParseInt(os.Getenv("LIMITS_IMAGE_PREVIEW_MAX_WIDTH"), 10, 32)
if err != nil {
panic(err)
}
config.Limits.ImagePreviewMaxWidth = int(v)
}
if len(os.Getenv("LIMITS_IMAGE_PREVIEW_MAX_HEIGHT")) > 0 {
v, err := strconv.ParseInt(os.Getenv("LIMITS_IMAGE_PREVIEW_MAX_HEIGHT"), 10, 32)
if err != nil {
panic(err)
}
config.Limits.ImagePreviewMaxHeight = int(v)
}
if len(os.Getenv("LIMITS_LANGUAGE_SCORE_THRESHOLD")) > 0 {
v, err := strconv.ParseFloat(os.Getenv("LIMITS_LANGUAGE_SCORE_THRESHOLD"), 64)
if err != nil {
panic(err)
}
config.Limits.LanguageScoreThreshold = v
}
}

View File

@ -0,0 +1,29 @@
package config
type Config struct {
Port int
APIURL string
Security SecurityConfig
Limits LimitsConfig
S3 S3Config
}
type SecurityConfig struct {
APIKey string `json:"api_key"`
}
type LimitsConfig struct {
ExternalCommandTimeoutSeconds int
FileProcessingMaxSizeMB int
ImagePreviewMaxWidth int
ImagePreviewMaxHeight int
LanguageScoreThreshold float64
}
type S3Config struct {
URL string
AccessKey string
SecretKey string
Region string
Secure bool
}

View File

@ -0,0 +1,15 @@
package core
const (
PipelinePDF = "pdf"
PipelineOffice = "office"
PipelineImage = "image"
PipelineVideo = "video"
)
const (
SnapshotStatusNew = "new"
SnapshotStatusProcessing = "processing"
SnapshotStatusReady = "ready"
SnapshotStatusError = "error"
)

View File

@ -0,0 +1,43 @@
package core
type PipelineRunOptions struct {
FileID string `json:"fileId"`
SnapshotID string `json:"snapshotId"`
Bucket string `json:"bucket"`
Key string `json:"key"`
}
type SnapshotUpdateOptions struct {
Options PipelineRunOptions `json:"options,omitempty"`
Original *S3Object `json:"original,omitempty"`
Preview *S3Object `json:"preview,omitempty"`
Text *S3Object `json:"text,omitempty"`
Thumbnail *ImageBase64 `json:"thumbnail,omitempty"`
Status string `json:"status,omitempty"`
}
type S3Object struct {
Bucket string `json:"bucket"`
Key string `json:"key"`
Size int64 `json:"size"`
Image *ImageProps `json:"image,omitempty"`
}
type ImageProps struct {
Width int `json:"width"`
Height int `json:"height"`
}
type ImageBase64 struct {
Base64 string `json:"base64"`
Width int `json:"width"`
Height int `json:"height"`
}
type Pipeline interface {
Run(PipelineRunOptions) error
}
type Builder interface {
Build(PipelineRunOptions) error
}

View File

@ -0,0 +1,48 @@
module voltaserve
go 1.21
toolchain go1.22.2
require (
github.com/gabriel-vasile/mimetype v1.4.3
github.com/gofiber/fiber/v2 v2.52.4
github.com/google/uuid v1.6.0
github.com/joho/godotenv v1.5.1
github.com/minio/minio-go/v7 v7.0.69
github.com/speps/go-hashids/v2 v2.0.1
go.uber.org/zap v1.27.0
)
require (
github.com/andybalholm/brotli v1.1.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.17.8 // indirect
github.com/klauspost/cpuid/v2 v2.2.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/minio/sha256-simd v1.0.1 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/philhofer/fwd v1.1.2 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/rs/xid v1.5.0 // indirect
github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94 // indirect
github.com/savsgio/gotils v0.0.0-20240303185622-093b76447511 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/stretchr/testify v1.8.4 // indirect
github.com/tinylib/msgp v1.1.9 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fasthttp v1.52.0 // indirect
github.com/valyala/tcplisten v1.0.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/crypto v0.22.0 // indirect
golang.org/x/net v0.24.0 // indirect
golang.org/x/sys v0.19.0 // indirect
golang.org/x/text v0.14.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
)

View File

@ -0,0 +1,179 @@
github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
github.com/gofiber/fiber/v2 v2.46.0 h1:wkkWotblsGVlLjXj2dpgKQAYHtXumsK/HyFugQM68Ns=
github.com/gofiber/fiber/v2 v2.46.0/go.mod h1:DNl0/c37WLe0g92U6lx1VMQuxGUQY5V7EIaVoEsUffc=
github.com/gofiber/fiber/v2 v2.52.4 h1:P+T+4iK7VaqUsq2PALYEfBBo6bJZ4q3FP8cZ84EggTM=
github.com/gofiber/fiber/v2 v2.52.4/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI=
github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg=
github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM=
github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.56 h1:pkZplIEHu8vinjkmhsexcXpWth2tjVLphrTZx6fBVZY=
github.com/minio/minio-go/v7 v7.0.56/go.mod h1:NUDy4A4oXPq1l2yK6LTSvCEzAMeIcoz9lcj5dbzSrRE=
github.com/minio/minio-go/v7 v7.0.69 h1:l8AnsQFyY1xiwa/DaQskY4NXSLA2yrGsW5iD9nRPVS0=
github.com/minio/minio-go/v7 v7.0.69/go.mod h1:XAvOPJQ5Xlzk5o3o/ArO2NMbhSGkimC+bpW/ngRKDmQ=
github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw=
github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94 h1:rmMl4fXJhKMNWl+K+r/fq4FbbKI+Ia2m9hYBLm2h4G4=
github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94/go.mod h1:90zrgN3D/WJsDd1iXHT96alCoN2KJo6/4x1DZC3wZs8=
github.com/savsgio/gotils v0.0.0-20220530130905-52f3993e8d6d/go.mod h1:Gy+0tqhJvgGlqnTF8CVGP0AaGRjwBtXs/a5PA0Y3+A4=
github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee h1:8Iv5m6xEo1NR1AvpV+7XmhI4r39LGNzwUL4YpMuL5vk=
github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee/go.mod h1:qwtSXrKuJh/zsFQ12yEE89xfCrGKK63Rr7ctU/uCo4g=
github.com/savsgio/gotils v0.0.0-20240303185622-093b76447511 h1:KanIMPX0QdEdB4R3CiimCAbxFrhB3j7h0/OvpYGVQa8=
github.com/savsgio/gotils v0.0.0-20240303185622-093b76447511/go.mod h1:sM7Mt7uEoCeFSCBM+qBrqvEo+/9vdmj19wzp3yzUhmg=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/speps/go-hashids/v2 v2.0.1 h1:ViWOEqWES/pdOSq+C1SLVa8/Tnsd52XC34RY7lt7m4g=
github.com/speps/go-hashids/v2 v2.0.1/go.mod h1:47LKunwvDZki/uRVD6NImtyk712yFzIs3UF3KlHohGw=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/tinylib/msgp v1.1.6/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOMufyw=
github.com/tinylib/msgp v1.1.8 h1:FCXC1xanKO4I8plpHGH2P7koL/RzZs12l/+r7vakfm0=
github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw=
github.com/tinylib/msgp v1.1.9 h1:SHf3yoO2sGA0veCJeCBYLHuttAVFHGm2RHgNodW7wQU=
github.com/tinylib/msgp v1.1.9/go.mod h1:BCXGB54lDD8qUEPmiG0cQQUANC4IUQyB2ItS2UDlO/k=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.47.0 h1:y7moDoxYzMooFpT5aHgNgVOQDrS3qlkfiP9mDtGGK9c=
github.com/valyala/fasthttp v1.47.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA=
github.com/valyala/fasthttp v1.52.0 h1:wqBQpxH71XW0e2g+Og4dzQM8pk34aFYlA1Ga8db7gU0=
github.com/valyala/fasthttp v1.52.0/go.mod h1:hf5C4QnVMkNXMspnsUlfM3WitlgYflyhHYoKol/szxQ=
github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g=
golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0=
golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

View File

@ -0,0 +1,24 @@
package helper
import (
"encoding/base64"
"net/http"
"os"
)
func ImageToBase64(path string) (string, error) {
bytes, err := os.ReadFile(path)
if err != nil {
return "", nil
}
var res string
mimeType := http.DetectContentType(bytes)
switch mimeType {
case "image/jpeg":
res += "data:image/jpeg;base64,"
case "image/png":
res += "data:image/png;base64,"
}
res += base64.StdEncoding.EncodeToString(bytes)
return res, nil
}

View File

@ -0,0 +1,22 @@
package helper
import (
"time"
"github.com/google/uuid"
"github.com/speps/go-hashids/v2"
)
func NewID() string {
hd := hashids.NewData()
hd.Salt = uuid.NewString()
h, err := hashids.NewWithData(hd)
if err != nil {
panic(err)
}
id, err := h.EncodeInt64([]int64{time.Now().UTC().UnixNano()})
if err != nil {
panic(err)
}
return id
}

View File

@ -0,0 +1,11 @@
package helper
import "github.com/gabriel-vasile/mimetype"
func DetectMimeFromFile(path string) string {
mime, err := mimetype.DetectFile(path)
if err != nil {
return "application/octet-stream"
}
return mime.String()
}

View File

@ -0,0 +1,14 @@
package helper
import (
"regexp"
"strings"
)
func RemoveNonNumeric(s string) string {
reg, err := regexp.Compile("[^0-9]+")
if err != nil {
return ""
}
return strings.TrimSpace(reg.ReplaceAllString(s, ""))
}

View File

@ -0,0 +1,147 @@
package identifier
import (
"path/filepath"
"strings"
)
type FileIdentifier struct {
}
func NewFileIdentifier() *FileIdentifier {
return &FileIdentifier{}
}
func (fi *FileIdentifier) IsPDF(path string) bool {
return strings.ToLower(filepath.Ext(path)) == ".pdf"
}
func (fi *FileIdentifier) IsOffice(path string) bool {
extensions := []string{
".xls",
".doc",
".ppt",
".xlsx",
".docx",
".pptx",
".odt",
".ott",
".ods",
".ots",
".odp",
".otp",
".odg",
".otg",
".odf",
".odc",
".rtf",
}
extension := filepath.Ext(path)
for _, v := range extensions {
if strings.ToLower(extension) == v {
return true
}
}
return false
}
func (fi *FileIdentifier) IsPlainText(path string) bool {
extensions := []string{
".txt",
".html",
".js",
"jsx",
".ts",
".tsx",
".css",
".sass",
".scss",
".go",
".py",
".rb",
".java",
".c",
".h",
".cpp",
".hpp",
".json",
".yml",
".yaml",
".toml",
".md",
}
extension := filepath.Ext(path)
for _, v := range extensions {
if strings.ToLower(extension) == v {
return true
}
}
return false
}
func (fi *FileIdentifier) IsImage(path string) bool {
extensions := []string{
".xpm",
".png",
".jpg",
".jpeg",
".jp2",
".gif",
".webp",
".tiff",
".bmp",
".ico",
".heif",
".xcf",
".svg",
}
extension := filepath.Ext(path)
for _, v := range extensions {
if strings.ToLower(extension) == v {
return true
}
}
return false
}
func (fi *FileIdentifier) IsNonAlphaChannelImage(path string) bool {
extensions := []string{
".jpg",
".jpeg",
".gif",
".tiff",
".bmp",
}
extension := filepath.Ext(path)
for _, v := range extensions {
if strings.ToLower(extension) == v {
return true
}
}
return false
}
func (fi *FileIdentifier) IsVideo(path string) bool {
extensions := []string{
".ogv",
".mpeg",
".mov",
".mqv",
".mp4",
".webm",
".3gp",
".3g2",
".avi",
".flv",
".mkv",
".asf",
".m4v",
}
extension := filepath.Ext(path)
for _, v := range extensions {
if strings.ToLower(extension) == v {
return true
}
}
return false
}

View File

@ -0,0 +1,28 @@
package identifier
import (
"voltaserve/core"
)
type PipelineIdentifier struct {
fileIdent *FileIdentifier
}
func NewPipelineIdentifier() *PipelineIdentifier {
return &PipelineIdentifier{
fileIdent: NewFileIdentifier(),
}
}
func (pi *PipelineIdentifier) Identify(opts core.PipelineRunOptions) string {
if pi.fileIdent.IsPDF(opts.Key) {
return core.PipelinePDF
} else if pi.fileIdent.IsOffice(opts.Key) || pi.fileIdent.IsPlainText(opts.Key) {
return core.PipelineOffice
} else if pi.fileIdent.IsImage(opts.Key) {
return core.PipelineImage
} else if pi.fileIdent.IsVideo(opts.Key) {
return core.PipelineVideo
}
return ""
}

View File

@ -0,0 +1,72 @@
package infra
import (
"bytes"
"errors"
"os/exec"
"sync"
"time"
"voltaserve/config"
)
var commandMutex sync.Mutex
type Command struct {
config config.Config
}
func NewCommand() *Command {
return &Command{config: config.GetConfig()}
}
func (r *Command) Exec(name string, arg ...string) error {
commandMutex.Lock()
defer commandMutex.Unlock()
timeout := time.Duration(r.config.Limits.ExternalCommandTimeoutSeconds) * time.Second
cmd := exec.Command(name, arg...)
var stderr bytes.Buffer
cmd.Stderr = &stderr
if err := cmd.Start(); err != nil {
return err
}
timer := time.AfterFunc(timeout, func() {
_ = cmd.Process.Kill()
})
if err := cmd.Wait(); err != nil {
if stderr.Len() > 0 {
return errors.New(stderr.String())
} else {
return err
}
}
timer.Stop()
return nil
}
func (r *Command) ReadOutput(name string, arg ...string) (string, error) {
commandMutex.Lock()
defer commandMutex.Unlock()
timeout := time.Duration(r.config.Limits.ExternalCommandTimeoutSeconds) * time.Second
cmd := exec.Command(name, arg...)
var stderr bytes.Buffer
cmd.Stderr = &stderr
res, err := cmd.Output()
if err != nil {
if stderr.Len() > 0 {
return "", errors.New(stderr.String())
} else {
return "", err
}
}
timer := time.AfterFunc(timeout, func() {
_ = cmd.Process.Kill()
})
timer.Stop()
return string(res), nil
}

View File

@ -0,0 +1,30 @@
package infra
import (
"fmt"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
)
var StrScheduler = fmt.Sprintf("%-13s", "scheduler")
var StrPipeline = fmt.Sprintf("%-13s", "pipeline")
var StrBuilder = fmt.Sprintf("%-13s", "builder")
var StrWorkerStatus = fmt.Sprintf("%-13s", "worker_status")
var StrQueueStatus = fmt.Sprintf("%-13s", "queue_status")
var logger *zap.SugaredLogger
func GetLogger() (*zap.SugaredLogger, error) {
if logger == nil {
config := zap.NewDevelopmentConfig()
config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
config.DisableCaller = true
if l, err := config.Build(); err != nil {
return nil, err
} else {
logger = l.Sugar()
}
}
return logger, nil
}

View File

@ -0,0 +1,205 @@
package infra
import (
"bytes"
"context"
"errors"
"io"
"strings"
"voltaserve/config"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/minio/minio-go/v7/pkg/sse"
)
type S3Manager struct {
config config.S3Config
client *minio.Client
}
func NewS3Manager() *S3Manager {
mgr := new(S3Manager)
mgr.config = config.GetConfig().S3
return mgr
}
func (mgr *S3Manager) GetFile(objectName string, filePath string, bucketName string) error {
if mgr.client == nil {
if err := mgr.connect(); err != nil {
return err
}
}
if err := mgr.client.FGetObject(context.Background(), bucketName, objectName, filePath, minio.GetObjectOptions{}); err != nil {
return err
}
return nil
}
func (mgr *S3Manager) PutFile(objectName string, filePath string, contentType string, bucketName string) error {
if mgr.client == nil {
if err := mgr.connect(); err != nil {
return err
}
}
if contentType == "" {
contentType = "application/octet-stream"
}
if _, err := mgr.client.FPutObject(context.Background(), bucketName, objectName, filePath, minio.PutObjectOptions{
ContentType: contentType,
}); err != nil {
return err
}
return nil
}
func (mgr *S3Manager) PutText(objectName string, text string, contentType string, bucketName string) error {
if contentType != "" && contentType != "text/plain" && contentType != "application/json" {
return errors.New("invalid content type '" + contentType + "'")
}
if contentType == "" {
contentType = "text/plain"
}
if mgr.client == nil {
if err := mgr.connect(); err != nil {
return err
}
}
if _, err := mgr.client.PutObject(context.Background(), bucketName, objectName, strings.NewReader(text), int64(len(text)), minio.PutObjectOptions{
ContentType: contentType,
}); err != nil {
return err
}
return nil
}
func (mgr *S3Manager) GetObject(objectName string, bucketName string) (*bytes.Buffer, error) {
if mgr.client == nil {
if err := mgr.connect(); err != nil {
return nil, err
}
}
reader, err := mgr.client.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
if err != nil {
return nil, err
}
var buf bytes.Buffer
_, err = io.Copy(io.Writer(&buf), reader)
if err != nil {
return nil, nil
}
return &buf, nil
}
func (mgr *S3Manager) GetText(objectName string, bucketName string) (string, error) {
if mgr.client == nil {
if err := mgr.connect(); err != nil {
return "", err
}
}
reader, err := mgr.client.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
if err != nil {
return "", err
}
buf := new(strings.Builder)
_, err = io.Copy(buf, reader)
if err != nil {
return "", nil
}
return buf.String(), nil
}
func (mgr *S3Manager) RemoveObject(objectName string, bucketName string) error {
if mgr.client == nil {
if err := mgr.connect(); err != nil {
return err
}
}
err := mgr.client.RemoveObject(context.Background(), bucketName, objectName, minio.RemoveObjectOptions{})
if err != nil {
return err
}
return nil
}
func (mgr *S3Manager) CreateBucket(bucketName string) error {
if mgr.client == nil {
if err := mgr.connect(); err != nil {
return err
}
}
found, err := mgr.client.BucketExists(context.Background(), bucketName)
if err != nil {
return err
}
if !found {
if err = mgr.client.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{
Region: mgr.config.Region,
}); err != nil {
return err
}
}
return nil
}
func (mgr *S3Manager) RemoveBucket(bucketName string) error {
if mgr.client == nil {
if err := mgr.connect(); err != nil {
return err
}
}
found, err := mgr.client.BucketExists(context.Background(), bucketName)
if err != nil {
return err
}
if !found {
return nil
}
objectCh := mgr.client.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{
Prefix: "",
Recursive: true,
})
mgr.client.RemoveObjects(context.Background(), bucketName, objectCh, minio.RemoveObjectsOptions{})
if err = mgr.client.RemoveBucket(context.Background(), bucketName); err != nil {
return err
}
return nil
}
func (mgr *S3Manager) EnableBucketEncryption(bucketName string) error {
if mgr.client == nil {
if err := mgr.connect(); err != nil {
return err
}
}
err := mgr.client.SetBucketEncryption(context.Background(), bucketName, sse.NewConfigurationSSES3())
if err != nil {
return err
}
return nil
}
func (mgr *S3Manager) DisableBucketEncryption(bucketName string) error {
if mgr.client == nil {
if err := mgr.connect(); err != nil {
return err
}
}
err := mgr.client.RemoveBucketEncryption(context.Background(), bucketName)
if err != nil {
return err
}
return nil
}
func (mgr *S3Manager) connect() error {
client, err := minio.New(mgr.config.URL, &minio.Options{
Creds: credentials.NewStaticV4(mgr.config.AccessKey, mgr.config.SecretKey, ""),
Secure: mgr.config.Secure,
})
if err != nil {
return err
}
mgr.client = client
return nil
}

View File

@ -0,0 +1,73 @@
package main
import (
"errors"
"flag"
"fmt"
"net/http"
"os"
"voltaserve/config"
"voltaserve/core"
"voltaserve/runtime"
"github.com/gofiber/fiber/v2"
"github.com/joho/godotenv"
)
func main() {
if _, err := os.Stat(".env.local"); err == nil {
err := godotenv.Load(".env.local")
if err != nil {
panic(err)
}
} else {
err := godotenv.Load()
if err != nil {
panic(err)
}
}
cfg := config.GetConfig()
schedulerOpts := runtime.NewDefaultSchedulerOptions()
pipelineWorkers := flag.Int("pipeline-workers", schedulerOpts.PipelineWorkerCount, "Number of pipeline workers")
flag.Parse()
scheduler := runtime.NewScheduler(runtime.SchedulerOptions{
PipelineWorkerCount: *pipelineWorkers,
})
app := fiber.New()
app.Get("v1/health", func(c *fiber.Ctx) error {
return c.SendStatus(200)
})
app.Post("v1/pipelines/run", func(c *fiber.Ctx) error {
apiKey := c.Query("api_key")
if apiKey == "" {
if err := c.SendStatus(http.StatusBadRequest); err != nil {
return err
}
return errors.New("missing query param api_key")
}
if apiKey != cfg.Security.APIKey {
if err := c.SendStatus(http.StatusUnauthorized); err != nil {
return err
}
return errors.New("invalid api_key")
}
opts := new(core.PipelineRunOptions)
if err := c.BodyParser(opts); err != nil {
return err
}
scheduler.SchedulePipeline(opts)
return c.SendStatus(200)
})
scheduler.Start()
if err := app.Listen(fmt.Sprintf(":%d", cfg.Port)); err != nil {
panic(err)
}
}

View File

@ -0,0 +1,73 @@
package pipeline
import (
"errors"
"voltaserve/client"
"voltaserve/core"
"voltaserve/identifier"
)
type Dispatcher struct {
pipelineIdentifier *identifier.PipelineIdentifier
pdfPipeline core.Pipeline
imagePipeline core.Pipeline
officePipeline core.Pipeline
videoPipeline core.Pipeline
apiClient *client.APIClient
}
func NewDispatcher() *Dispatcher {
return &Dispatcher{
pipelineIdentifier: identifier.NewPipelineIdentifier(),
pdfPipeline: NewPDFPipeline(),
imagePipeline: NewImagePipeline(),
officePipeline: NewOfficePipeline(),
videoPipeline: NewVideoPipeline(),
apiClient: client.NewAPIClient(),
}
}
func (d *Dispatcher) Dispatch(opts core.PipelineRunOptions) error {
if err := d.apiClient.UpdateSnapshot(core.SnapshotUpdateOptions{
Options: opts,
Status: core.SnapshotStatusProcessing,
}); err != nil {
return err
}
p := d.pipelineIdentifier.Identify(opts)
var err error
if p == core.PipelinePDF {
err = d.pdfPipeline.Run(opts)
} else if p == core.PipelineOffice {
err = d.officePipeline.Run(opts)
} else if p == core.PipelineImage {
err = d.imagePipeline.Run(opts)
} else if p == core.PipelineVideo {
err = d.videoPipeline.Run(opts)
} else {
if err := d.apiClient.UpdateSnapshot(core.SnapshotUpdateOptions{
Options: opts,
Status: core.SnapshotStatusError,
}); err != nil {
return err
}
return errors.New("no matching pipeline found")
}
if err != nil {
if err := d.apiClient.UpdateSnapshot(core.SnapshotUpdateOptions{
Options: opts,
Status: core.SnapshotStatusError,
}); err != nil {
return err
}
return nil
} else {
if err := d.apiClient.UpdateSnapshot(core.SnapshotUpdateOptions{
Options: opts,
Status: core.SnapshotStatusReady,
}); err != nil {
return err
}
return nil
}
}

View File

@ -0,0 +1,99 @@
package pipeline
import (
"os"
"path/filepath"
"voltaserve/client"
"voltaserve/config"
"voltaserve/core"
"voltaserve/helper"
"voltaserve/identifier"
"voltaserve/infra"
"voltaserve/processor"
"go.uber.org/zap"
)
type imagePipeline struct {
imageProc *processor.ImageProcessor
s3 *infra.S3Manager
apiClient *client.APIClient
fileIdent *identifier.FileIdentifier
logger *zap.SugaredLogger
config config.Config
}
func NewImagePipeline() core.Pipeline {
logger, err := infra.GetLogger()
if err != nil {
panic(err)
}
return &imagePipeline{
imageProc: processor.NewImageProcessor(),
s3: infra.NewS3Manager(),
apiClient: client.NewAPIClient(),
fileIdent: identifier.NewFileIdentifier(),
logger: logger,
config: config.GetConfig(),
}
}
func (p *imagePipeline) Run(opts core.PipelineRunOptions) error {
inputPath := filepath.FromSlash(os.TempDir() + "/" + helper.NewID() + filepath.Ext(opts.Key))
if err := p.s3.GetFile(opts.Key, inputPath, opts.Bucket); err != nil {
return err
}
stat, err := os.Stat(inputPath)
if err != nil {
return err
}
imageProps, err := p.imageProc.MeasureImage(inputPath)
if err != nil {
return err
}
updateOpts := core.SnapshotUpdateOptions{
Options: opts,
Original: &core.S3Object{
Bucket: opts.Bucket,
Key: opts.Key,
Image: &imageProps,
Size: stat.Size(),
},
}
if filepath.Ext(inputPath) == ".tiff" {
jpegPath := filepath.FromSlash(os.TempDir() + "/" + helper.NewID() + ".jpg")
if err := p.imageProc.ConvertImage(inputPath, jpegPath); err != nil {
return err
}
thumbnail, err := p.imageProc.Base64Thumbnail(jpegPath)
if err != nil {
return err
}
updateOpts.Thumbnail = &thumbnail
updateOpts.Preview = &core.S3Object{
Bucket: opts.Bucket,
Key: opts.FileID + "/" + opts.SnapshotID + "/preview.jpg",
Size: stat.Size(),
}
if err := p.s3.PutFile(updateOpts.Preview.Key, jpegPath, helper.DetectMimeFromFile(jpegPath), updateOpts.Preview.Bucket); err != nil {
return err
}
if err := os.Remove(inputPath); err != nil {
return err
}
inputPath = jpegPath
} else {
thumbnail, err := p.imageProc.Base64Thumbnail(inputPath)
if err != nil {
return err
}
updateOpts.Thumbnail = &thumbnail
}
if err := p.apiClient.UpdateSnapshot(updateOpts); err != nil {
return err
}
if err := os.Remove(inputPath); err != nil {
return err
}
return nil
}

View File

@ -0,0 +1,86 @@
package pipeline
import (
"os"
"path/filepath"
"voltaserve/client"
"voltaserve/config"
"voltaserve/core"
"voltaserve/helper"
"voltaserve/infra"
"voltaserve/processor"
)
type officePipeline struct {
pdfPipeline core.Pipeline
officeProc *processor.OfficeProcessor
pdfProc *processor.PDFProcessor
s3 *infra.S3Manager
config config.Config
apiClient *client.APIClient
}
func NewOfficePipeline() core.Pipeline {
return &officePipeline{
pdfPipeline: NewPDFPipeline(),
officeProc: processor.NewOfficeProcessor(),
pdfProc: processor.NewPDFProcessor(),
s3: infra.NewS3Manager(),
config: config.GetConfig(),
apiClient: client.NewAPIClient(),
}
}
func (p *officePipeline) Run(opts core.PipelineRunOptions) error {
inputPath := filepath.FromSlash(os.TempDir() + "/" + helper.NewID() + filepath.Ext(opts.Key))
if err := p.s3.GetFile(opts.Key, inputPath, opts.Bucket); err != nil {
return err
}
outputPath, err := p.officeProc.PDF(inputPath)
if err != nil {
return err
}
stat, err := os.Stat(outputPath)
if err != nil {
return err
}
thumbnail, err := p.pdfProc.Base64Thumbnail(outputPath)
if err != nil {
return err
}
if err := p.apiClient.UpdateSnapshot(core.SnapshotUpdateOptions{
Options: opts,
Thumbnail: &thumbnail,
}); err != nil {
return err
}
previewKey := opts.FileID + "/" + opts.SnapshotID + "/preview.pdf"
if err := p.s3.PutFile(previewKey, outputPath, helper.DetectMimeFromFile(outputPath), opts.Bucket); err != nil {
return err
}
if err := p.apiClient.UpdateSnapshot(core.SnapshotUpdateOptions{
Options: opts,
Preview: &core.S3Object{
Bucket: opts.Bucket,
Key: previewKey,
Size: stat.Size(),
},
}); err != nil {
return err
}
if err := p.pdfPipeline.Run(core.PipelineRunOptions{
Bucket: opts.Bucket,
Key: previewKey,
FileID: opts.FileID,
SnapshotID: opts.SnapshotID,
}); err != nil {
return err
}
if err := os.Remove(inputPath); err != nil {
return err
}
if err := os.Remove(outputPath); err != nil {
return err
}
return nil
}

View File

@ -0,0 +1,82 @@
package pipeline
import (
"os"
"path/filepath"
"voltaserve/client"
"voltaserve/config"
"voltaserve/core"
"voltaserve/helper"
"voltaserve/identifier"
"voltaserve/infra"
"voltaserve/processor"
"go.uber.org/zap"
)
type pdfPipeline struct {
pdfProc *processor.PDFProcessor
imageProc *processor.ImageProcessor
s3 *infra.S3Manager
apiClient *client.APIClient
fileIdent *identifier.FileIdentifier
logger *zap.SugaredLogger
config config.Config
}
func NewPDFPipeline() core.Pipeline {
logger, err := infra.GetLogger()
if err != nil {
panic(err)
}
return &pdfPipeline{
pdfProc: processor.NewPDFProcessor(),
imageProc: processor.NewImageProcessor(),
s3: infra.NewS3Manager(),
apiClient: client.NewAPIClient(),
fileIdent: identifier.NewFileIdentifier(),
logger: logger,
config: config.GetConfig(),
}
}
func (p *pdfPipeline) Run(opts core.PipelineRunOptions) error {
inputPath := filepath.FromSlash(os.TempDir() + "/" + helper.NewID() + filepath.Ext(opts.Key))
if err := p.s3.GetFile(opts.Key, inputPath, opts.Bucket); err != nil {
return err
}
thumbnail, err := p.pdfProc.Base64Thumbnail(inputPath)
if err != nil {
return err
}
if err := p.apiClient.UpdateSnapshot(core.SnapshotUpdateOptions{
Options: opts,
Thumbnail: &thumbnail,
}); err != nil {
return err
}
text, err := p.pdfProc.TextFromPDF(inputPath)
if err != nil {
p.logger.Named(infra.StrPipeline).Errorw(err.Error())
}
textKey := opts.FileID + "/" + opts.SnapshotID + "/text.txt"
if text != "" && err == nil {
if err := p.s3.PutText(textKey, text, "text/plain", opts.Bucket); err != nil {
return err
}
}
if err := p.apiClient.UpdateSnapshot(core.SnapshotUpdateOptions{
Options: opts,
Text: &core.S3Object{
Bucket: opts.Bucket,
Key: textKey,
Size: int64(len(text)),
},
}); err != nil {
return err
}
if err := os.Remove(inputPath); err != nil {
return err
}
return nil
}

View File

@ -0,0 +1,49 @@
package pipeline
import (
"os"
"path/filepath"
"voltaserve/client"
"voltaserve/core"
"voltaserve/helper"
"voltaserve/identifier"
"voltaserve/infra"
"voltaserve/processor"
)
type videoPipeline struct {
pipelineIdentifier *identifier.PipelineIdentifier
videoProc *processor.VideoProcessor
s3 *infra.S3Manager
apiClient *client.APIClient
}
func NewVideoPipeline() core.Pipeline {
return &videoPipeline{
pipelineIdentifier: identifier.NewPipelineIdentifier(),
videoProc: processor.NewVideoProcessor(),
s3: infra.NewS3Manager(),
apiClient: client.NewAPIClient(),
}
}
func (p *videoPipeline) Run(opts core.PipelineRunOptions) error {
inputPath := filepath.FromSlash(os.TempDir() + "/" + helper.NewID() + filepath.Ext(opts.Key))
if err := p.s3.GetFile(opts.Key, inputPath, opts.Bucket); err != nil {
return err
}
thumbnail, err := p.videoProc.Base64Thumbnail(inputPath)
if err != nil {
return err
}
if err := p.apiClient.UpdateSnapshot(core.SnapshotUpdateOptions{
Options: opts,
Thumbnail: &thumbnail,
}); err != nil {
return err
}
if err := os.Remove(inputPath); err != nil {
return err
}
return nil
}

View File

@ -0,0 +1,137 @@
package processor
import (
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"voltaserve/client"
"voltaserve/config"
"voltaserve/core"
"voltaserve/helper"
"voltaserve/identifier"
"voltaserve/infra"
)
type ImageProcessor struct {
apiClient *client.APIClient
fileIdent *identifier.FileIdentifier
config config.Config
}
func NewImageProcessor() *ImageProcessor {
return &ImageProcessor{
apiClient: client.NewAPIClient(),
fileIdent: identifier.NewFileIdentifier(),
config: config.GetConfig(),
}
}
func (p *ImageProcessor) Base64Thumbnail(inputPath string) (core.ImageBase64, error) {
inputSize, err := p.MeasureImage(inputPath)
if err != nil {
return core.ImageBase64{}, err
}
if inputSize.Width > p.config.Limits.ImagePreviewMaxWidth || inputSize.Height > p.config.Limits.ImagePreviewMaxHeight {
outputPath := filepath.FromSlash(os.TempDir() + "/" + helper.NewID() + filepath.Ext(inputPath))
if inputSize.Width > inputSize.Height {
if err := p.ResizeImage(inputPath, p.config.Limits.ImagePreviewMaxWidth, 0, outputPath); err != nil {
return core.ImageBase64{}, err
}
} else {
if err := p.ResizeImage(inputPath, 0, p.config.Limits.ImagePreviewMaxHeight, outputPath); err != nil {
return core.ImageBase64{}, err
}
}
b64, err := helper.ImageToBase64(outputPath)
if err != nil {
return core.ImageBase64{}, err
}
size, err := p.MeasureImage(outputPath)
if err != nil {
return core.ImageBase64{}, err
}
return core.ImageBase64{
Base64: b64,
Width: size.Width,
Height: size.Height,
}, nil
} else {
b64, err := helper.ImageToBase64(inputPath)
if err != nil {
return core.ImageBase64{}, err
}
size, err := p.MeasureImage(inputPath)
if err != nil {
return core.ImageBase64{}, err
}
return core.ImageBase64{
Base64: b64,
Width: size.Width,
Height: size.Height,
}, nil
}
}
func (p *ImageProcessor) MeasureImage(inputPath string) (core.ImageProps, error) {
size, err := infra.NewCommand().ReadOutput("identify", "-format", "%w,%h", inputPath)
if err != nil {
return core.ImageProps{}, err
}
values := strings.Split(size, ",")
width, err := strconv.Atoi(helper.RemoveNonNumeric(values[0]))
if err != nil {
return core.ImageProps{}, err
}
height, err := strconv.Atoi(helper.RemoveNonNumeric(values[1]))
if err != nil {
return core.ImageProps{}, err
}
return core.ImageProps{Width: width, Height: height}, nil
}
func (p *ImageProcessor) ResizeImage(inputPath string, width int, height int, outputPath string) error {
var widthStr string
if width == 0 {
widthStr = ""
} else {
widthStr = strconv.FormatInt(int64(width), 10)
}
var heightStr string
if height == 0 {
heightStr = ""
} else {
heightStr = strconv.FormatInt(int64(height), 10)
}
if err := infra.NewCommand().Exec("convert", "-resize", widthStr+"x"+heightStr, inputPath, outputPath); err != nil {
return err
}
return nil
}
func (p *ImageProcessor) ThumbnailFromImage(inputPath string, width int, height int, outputPath string) error {
var widthStr string
if width == 0 {
widthStr = ""
} else {
widthStr = strconv.FormatInt(int64(width), 10)
}
var heightStr string
if height == 0 {
heightStr = ""
} else {
heightStr = strconv.FormatInt(int64(height), 10)
}
if err := infra.NewCommand().Exec("convert", "-thumbnail", widthStr+"x"+heightStr, "-background", "white", "-alpha", "remove", "-flatten", fmt.Sprintf("%s[0]", inputPath), outputPath); err != nil {
return err
}
return nil
}
func (p *ImageProcessor) ConvertImage(inputPath string, outputPath string) error {
if err := infra.NewCommand().Exec("convert", inputPath, outputPath); err != nil {
return err
}
return nil
}

View File

@ -0,0 +1,42 @@
package processor
import (
"go.uber.org/zap"
"os"
"path"
"path/filepath"
"strings"
"voltaserve/config"
"voltaserve/helper"
"voltaserve/infra"
)
type OfficeProcessor struct {
cmd *infra.Command
config config.Config
logger *zap.SugaredLogger
}
func NewOfficeProcessor() *OfficeProcessor {
logger, err := infra.GetLogger()
if err != nil {
panic(err)
}
return &OfficeProcessor{
cmd: infra.NewCommand(),
config: config.GetConfig(),
logger: logger,
}
}
func (p *OfficeProcessor) PDF(inputPath string) (string, error) {
outputDir := filepath.FromSlash(os.TempDir() + "/" + helper.NewID())
if err := infra.NewCommand().Exec("soffice", "--headless", "--convert-to", "pdf", "--outdir", outputDir, inputPath); err != nil {
return "", err
}
if _, err := os.Stat(inputPath); err != nil {
return "", err
}
base := filepath.Base(inputPath)
return filepath.FromSlash(outputDir + "/" + strings.TrimSuffix(base, path.Ext(base)) + ".pdf"), nil
}

View File

@ -0,0 +1,65 @@
package processor
import (
"os"
"path/filepath"
"strings"
"voltaserve/config"
"voltaserve/core"
"voltaserve/helper"
"voltaserve/infra"
)
type PDFProcessor struct {
cmd *infra.Command
imageProc *ImageProcessor
config config.Config
}
func NewPDFProcessor() *PDFProcessor {
return &PDFProcessor{
cmd: infra.NewCommand(),
imageProc: NewImageProcessor(),
config: config.GetConfig(),
}
}
func (p *PDFProcessor) Base64Thumbnail(inputPath string) (core.ImageBase64, error) {
outputPath := filepath.FromSlash(os.TempDir() + "/" + helper.NewID() + ".png")
if err := p.imageProc.ThumbnailFromImage(inputPath, 0, p.config.Limits.ImagePreviewMaxHeight, outputPath); err != nil {
return core.ImageBase64{}, err
}
b64, err := helper.ImageToBase64(outputPath)
if err != nil {
return core.ImageBase64{}, err
}
imageProps, err := p.imageProc.MeasureImage(outputPath)
if err != nil {
return core.ImageBase64{}, err
}
if _, err := os.Stat(outputPath); err == nil {
if err := os.Remove(outputPath); err != nil {
return core.ImageBase64{}, err
}
}
return core.ImageBase64{
Base64: b64,
Width: imageProps.Width,
Height: imageProps.Height,
}, nil
}
func (p *PDFProcessor) TextFromPDF(inputPath string) (string, error) {
outputPath := filepath.FromSlash(os.TempDir() + "/" + helper.NewID() + ".txt")
if err := infra.NewCommand().Exec("pdftotext", inputPath, outputPath); err != nil {
return "", err
}
b, err := os.ReadFile(outputPath)
if err != nil {
return "", err
}
if err := os.Remove(outputPath); err != nil {
return "", err
}
return strings.TrimSpace(string(b)), nil
}

View File

@ -0,0 +1,68 @@
package processor
import (
"go.uber.org/zap"
"os"
"path/filepath"
"voltaserve/config"
"voltaserve/core"
"voltaserve/helper"
"voltaserve/infra"
)
type VideoProcessor struct {
cmd *infra.Command
imageProc *ImageProcessor
config config.Config
logger *zap.SugaredLogger
}
func NewVideoProcessor() *VideoProcessor {
logger, err := infra.GetLogger()
if err != nil {
panic(err)
}
return &VideoProcessor{
cmd: infra.NewCommand(),
imageProc: NewImageProcessor(),
config: config.GetConfig(),
logger: logger,
}
}
func (p *VideoProcessor) Thumbnail(inputPath string, width int, height int, outputPath string) error {
tmpPath := filepath.FromSlash(os.TempDir() + "/" + helper.NewID() + ".png")
if err := infra.NewCommand().Exec("ffmpeg", "-i", inputPath, "-frames:v", "1", tmpPath); err != nil {
return err
}
if err := p.imageProc.ResizeImage(tmpPath, width, height, outputPath); err != nil {
return err
}
if err := os.Remove(tmpPath); err != nil {
return err
}
return nil
}
func (p *VideoProcessor) Base64Thumbnail(inputPath string) (core.ImageBase64, error) {
outputPath := filepath.FromSlash(os.TempDir() + "/" + helper.NewID() + ".png")
if err := p.Thumbnail(inputPath, 0, p.config.Limits.ImagePreviewMaxHeight, outputPath); err != nil {
return core.ImageBase64{}, err
}
b64, err := helper.ImageToBase64(outputPath)
if err != nil {
return core.ImageBase64{}, err
}
imageProps, err := p.imageProc.MeasureImage(outputPath)
if err != nil {
return core.ImageBase64{}, err
}
if err := os.Remove(outputPath); err != nil {
return core.ImageBase64{}, err
}
return core.ImageBase64{
Base64: b64,
Width: imageProps.Width,
Height: imageProps.Height,
}, nil
}

View File

@ -0,0 +1,129 @@
package runtime
import (
"runtime"
"time"
"voltaserve/pipeline"
"voltaserve/client"
"voltaserve/core"
"voltaserve/infra"
"go.uber.org/zap"
)
type Scheduler struct {
pipelineQueue [][]core.PipelineRunOptions
pipelineWorkerCount int
activePipelineCount int
apiClient *client.APIClient
logger *zap.SugaredLogger
}
type SchedulerOptions struct {
PipelineWorkerCount int
}
func NewDefaultSchedulerOptions() SchedulerOptions {
opts := SchedulerOptions{}
if runtime.NumCPU() == 1 {
opts.PipelineWorkerCount = 1
} else {
opts.PipelineWorkerCount = runtime.NumCPU()
}
return opts
}
func NewScheduler(opts SchedulerOptions) *Scheduler {
logger, err := infra.GetLogger()
if err != nil {
panic(err)
}
return &Scheduler{
pipelineQueue: make([][]core.PipelineRunOptions, opts.PipelineWorkerCount),
pipelineWorkerCount: opts.PipelineWorkerCount,
apiClient: client.NewAPIClient(),
logger: logger,
}
}
func (s *Scheduler) Start() {
s.logger.Named(infra.StrScheduler).Infow("🚀 launching", "type", "pipeline", "count", s.pipelineWorkerCount)
for i := 0; i < s.pipelineWorkerCount; i++ {
go s.pipelineWorker(i)
}
go s.pipelineQueueStatus()
go s.pipelineWorkerStatus()
}
func (s *Scheduler) SchedulePipeline(opts *core.PipelineRunOptions) {
index := 0
length := len(s.pipelineQueue[0])
for i := 0; i < s.pipelineWorkerCount; i++ {
if len(s.pipelineQueue[i]) < length {
index = i
length = len(s.pipelineQueue[i])
}
}
s.logger.Named(infra.StrScheduler).Infow("👉 choosing", "pipeline", index)
s.pipelineQueue[index] = append(s.pipelineQueue[index], *opts)
}
func (s *Scheduler) pipelineWorker(index int) {
dispatcher := pipeline.NewDispatcher()
s.pipelineQueue[index] = make([]core.PipelineRunOptions, 0)
s.logger.Named(infra.StrPipeline).Infow("⚙️ running", "worker", index)
for {
if len(s.pipelineQueue[index]) > 0 {
s.activePipelineCount++
opts := s.pipelineQueue[index][0]
s.logger.Named(infra.StrPipeline).Infow("🔨 working", "worker", index, "bucket", opts.Bucket, "key", opts.Key)
start := time.Now()
err := dispatcher.Dispatch(opts)
elapsed := time.Since(start)
if err == nil {
s.logger.Named(infra.StrPipeline).Infow("🎉 succeeded", "worker", index, "elapsed", elapsed, "bucket", opts.Bucket, "key", opts.Key)
} else {
s.logger.Named(infra.StrPipeline).Errorw("⛈️ failed", "worker", index, "elapsed", elapsed, "bucket", opts.Bucket, "key", opts.Key, "error", err.Error())
}
s.pipelineQueue[index] = s.pipelineQueue[index][1:]
s.activePipelineCount--
} else {
time.Sleep(500 * time.Millisecond)
}
}
}
func (s *Scheduler) pipelineQueueStatus() {
previous := -1
for {
time.Sleep(5 * time.Second)
sum := 0
for i := 0; i < s.pipelineWorkerCount; i++ {
sum += len(s.pipelineQueue[i])
}
if sum != previous {
if sum == 0 {
s.logger.Named(infra.StrQueueStatus).Infow("🌈 empty", "type", "pipeline")
} else {
s.logger.Named(infra.StrQueueStatus).Infow("⏳ items", "type", "pipeline", "count", sum)
}
}
previous = sum
}
}
func (s *Scheduler) pipelineWorkerStatus() {
previous := -1
for {
time.Sleep(3 * time.Second)
if previous != s.activePipelineCount {
if s.activePipelineCount == 0 {
s.logger.Named(infra.StrWorkerStatus).Infow("🌤️ all idle", "type", "pipeline")
} else {
s.logger.Named(infra.StrWorkerStatus).Infow("🔥 active", "type", "pipeline", "count", s.activePipelineCount)
}
}
previous = s.activePipelineCount
}
}

View File

@ -0,0 +1,21 @@
#!/bin/bash
zypper install -y ffmpeg-4
zypper install -y gawk ghostscript ImageMagick
# https://www.suse.com/support/kb/doc/?id=000019384
policy_path=$(identify -list policy | awk '/Path:/ {print $2}' | sed 's/\[built-in\]//') \
&& awk '/pattern="(PS|PS2|PS3|PDF|XPS|EPS|PCL)"/ { sub(/rights="write"/, "rights=\"read|write\"") } { print }' "$policy_path" > "$policy_path.tmp" \
&& mv "$policy_path.tmp" "$policy_path"
zypper install -y poppler-tools
zypper install -y \
libreoffice \
libreoffice-writer \
libreoffice-calc \
libreoffice-impress \
libreoffice-draw \
libreoffice-math \
libreoffice-base

View File

@ -0,0 +1,864 @@
#!/bin/bash
zypper install -y \
adinatha-fonts \
adobe-sourcecodepro-fonts \
adobe-sourcehansans-cn-fonts \
adobe-sourcehansans-hk-fonts \
adobe-sourcehansans-jp-fonts \
adobe-sourcehansans-kr-fonts \
adobe-sourcehansans-tw-fonts \
adobe-sourcehanserif-cn-fonts \
adobe-sourcehanserif-hk-fonts \
adobe-sourcehanserif-jp-fonts \
adobe-sourcehanserif-kr-fonts \
adobe-sourcehanserif-tw-fonts \
adobe-sourcesans3-fonts \
adobe-sourcesanspro-fonts \
adobe-sourceserif4-fonts \
adobe-sourceserifpro-fonts \
aldusleaf-crimson-text-fonts \
alee-fonts \
aliftype-amiri-fonts \
antijingoist-opendyslexic-fonts \
arabic-ae-fonts \
arabic-amiri-fonts \
arabic-bitmap-fonts \
arabic-fonts \
arabic-kacst-fonts \
arabic-kacstone-fonts \
arabic-naqsh-fonts \
arphic-bkai00mp-fonts \
arphic-bsmi00lp-fonts \
arphic-fonts \
arphic-gbsn00lp-fonts \
arphic-gkai00mp-fonts \
arphic-ukai-fonts \
arphic-uming-fonts \
autonym-fonts \
avesta-fonts \
babelstone-han-fonts \
babelstone-marchen-fonts \
babelstone-modern-fonts \
babelstone-ogham-fonts \
babelstone-phags-pa-fonts \
babelstone-runic-fonts \
baekmuk-bitmap-fonts \
baekmuk-ttf-fonts \
bitstream-vera-fonts \
blockzone-fonts \
bpg-fonts \
cadsondemak-fonts \
cantarell-fonts \
caslon-fonts \
cm-unicode-fonts \
consoleet-darwin-fonts \
consoleet-fixedsys-fonts \
consoleet-kbd-fonts \
consoleet-oldschoolpc-fonts \
consoleet-terminus-fonts \
consoleet-xorg-fonts \
courier-prime-fonts \
cozette-fonts \
cpmono_v07-fonts \
culmus-ancient-semitic-fonts \
culmus-fonts \
cyreal-alice-fonts \
cyreal-junge-fonts \
cyreal-lobster-cyrillic-fonts \
cyreal-lora-fonts \
cyreal-marko-horobchyk-fonts \
cyreal-marmelad-fonts \
cyreal-wire-fonts \
dai-banna-fonts \
dejavu-fonts \
delaguardo-inconsolata_lgc-fonts \
dina-bitmap-fonts \
eb-garamond-fonts \
eeyek-fonts \
efont-serif-fonts \
efont-unicode-bitmap-fonts \
farsi-fonts \
finalcut-bitmap-fonts \
fira-code-fonts \
fontawesome-fonts \
free-ttf-fonts \
gdouros-abydos-fonts \
gdouros-aegean-fonts \
gdouros-aegyptus-fonts \
gdouros-akkadian-fonts \
gdouros-alfios-fonts \
gdouros-analecta-fonts \
gdouros-anatolian-fonts \
gdouros-atavyros-fonts \
gdouros-maya-fonts \
gdouros-musica-fonts \
gdouros-symbola-fonts \
gdouros-text-fonts \
gdouros-unidings-fonts \
gnu-free-fonts \
gnu-unifont-bitmap-fonts \
gnu-unifont-legacy-bitmap-fonts \
google-alegreya-fonts \
google-alegreya-sans-fonts \
google-allerta-fonts \
google-anonymouspro-fonts \
google-arimo-fonts \
google-cabin-fonts \
google-caladea-fonts \
google-cardo-fonts \
google-carlito-fonts \
google-cousine-fonts \
google-croscore-fonts \
google-droid-fonts \
google-exo-fonts \
google-inconsolata-fonts \
google-lekton-fonts \
google-merriweather-fonts \
google-nobile-fonts \
google-opensans-fonts \
google-poppins-fonts \
google-roboto-fonts \
google-roboto-mono-fonts \
google-tinos-fonts \
google-worksans-fonts \
hack-fonts \
hartke-aurulentsans-fonts \
indic-fonts \
inter-fonts \
intlfonts-arabic-bitmap-fonts \
intlfonts-asian-bitmap-fonts \
intlfonts-bdf-fonts \
intlfonts-chinese-big-bitmap-fonts \
intlfonts-chinese-bitmap-fonts \
intlfonts-ethiopic-bitmap-fonts \
intlfonts-euro-bitmap-fonts \
intlfonts-japanese-big-bitmap-fonts \
intlfonts-japanese-bitmap-fonts \
intlfonts-phonetic-bitmap-fonts \
intlfonts-ttf-fonts \
intlfonts-type1-fonts \
iosevka-aile-fonts \
iosevka-curly-fonts \
iosevka-curly-slab-fonts \
iosevka-etoile-fonts \
iosevka-fonts \
iosevka-slab-fonts \
iosevka-ss01-fonts \
iosevka-ss02-fonts \
iosevka-ss03-fonts \
iosevka-ss04-fonts \
iosevka-ss05-fonts \
iosevka-ss06-fonts \
iosevka-ss07-fonts \
iosevka-ss08-fonts \
iosevka-ss09-fonts \
iosevka-ss10-fonts \
iosevka-ss11-fonts \
iosevka-ss12-fonts \
iosevka-ss13-fonts \
iosevka-ss14-fonts \
iosevka-ss15-fonts \
iosevka-ss16-fonts \
iosevka-ss17-fonts \
iosevka-ss18-fonts \
ipa-ex-gothic-fonts \
ipa-ex-mincho-fonts \
ipa-gothic-bold-fonts \
ipa-gothic-bolditalic-fonts \
ipa-gothic-fonts \
ipa-gothic-italic-fonts \
ipa-mincho-fonts \
ipa-pgothic-bold-fonts \
ipa-pgothic-bolditalic-fonts \
ipa-pgothic-fonts \
ipa-pgothic-italic-fonts \
ipa-pmincho-fonts \
ipa-uigothic-fonts \
jetbrains-mono-fonts \
jomolhari-fonts \
js-technology-fonts \
kde-oxygen-fonts \
khmeros-fonts \
kika-fixedsys-fonts \
lato-fonts \
liberation-fonts \
libertinus-fonts \
lilypond-emmentaler-fonts \
linux-libertine-fonts \
lklug-fonts \
lomt-blackout-fonts \
lomt-chunk-fonts \
lomt-fanwood-fonts \
lomt-goudybookletter-fonts \
lomt-junction-fonts \
lomt-knewave-fonts \
lomt-leaguegothic-fonts \
lomt-lindenhill-fonts \
lomt-orbitron-fonts \
lomt-ostrichsans-fonts \
lomt-prociono-fonts \
lomt-script1-fonts \
lomt-sniglet-fonts \
lomt-sortsmillgoudy-fonts \
lyx-fonts \
manchu-fonts \
mathgl-fonts \
mathjax-ams-fonts \
mathjax-caligraphic-fonts \
mathjax-fraktur-fonts \
mathjax-main-fonts \
mathjax-math-fonts \
mathjax-sansserif-fonts \
mathjax-script-fonts \
mathjax-size1-fonts \
mathjax-size2-fonts \
mathjax-size3-fonts \
mathjax-size4-fonts \
mathjax-typewriter-fonts \
mathjax-winchrome-fonts \
mathjax-winie6-fonts \
meslo-lg-fonts \
mgopen-fonts \
miao-fonts \
mikachan-fonts \
mingzat-fonts \
monapo-fonts \
mongolian-fonts \
motoya-lcedar-fonts \
motoya-lmaru-fonts \
mplus-code-latin-variable-fonts \
mplus-code-latin50-fonts \
mplus-code-latin60-fonts \
mplus-fonts \
mplus1-code-fonts \
mplus1-code-variable-fonts \
mplus1-fonts \
mplus1-variable-fonts \
mplus2-fonts \
mplus2-variable-fonts \
musescore-fonts \
namdhinggo-fonts \
nanum-fonts \
nanum-gothic-coding-fonts \
noto-coloremoji-fonts \
noto-kufiarabic-fonts \
noto-naskharabic-fonts \
noto-naskharabic-ui-fonts \
noto-nastaliqurdu-fonts \
noto-sans-adlam-fonts \
noto-sans-adlamunjoined-fonts \
noto-sans-anatolianhieroglyphs-fonts \
noto-sans-arabic-fonts \
noto-sans-arabic-ui-fonts \
noto-sans-armenian-fonts \
noto-sans-avestan-fonts \
noto-sans-balinese-fonts \
noto-sans-bamum-fonts \
noto-sans-batak-fonts \
noto-sans-bengali-fonts \
noto-sans-bengali-ui-fonts \
noto-sans-brahmi-fonts \
noto-sans-buginese-fonts \
noto-sans-buhid-fonts \
noto-sans-canadianaboriginal-fonts \
noto-sans-carian-fonts \
noto-sans-chakma-fonts \
noto-sans-cham-fonts \
noto-sans-cherokee-fonts \
noto-sans-cjk-fonts \
noto-sans-coptic-fonts \
noto-sans-cuneiform-fonts \
noto-sans-cypriot-fonts \
noto-sans-deseret-fonts \
noto-sans-devanagari-fonts \
noto-sans-devanagari-ui-fonts \
noto-sans-egyptianhieroglyphs-fonts \
noto-sans-ethiopic-fonts \
noto-sans-fonts \
noto-sans-georgian-fonts \
noto-sans-glagolitic-fonts \
noto-sans-gothic-fonts \
noto-sans-gujarati-fonts \
noto-sans-gujarati-ui-fonts \
noto-sans-gurmukhi-fonts \
noto-sans-gurmukhi-ui-fonts \
noto-sans-hanunoo-fonts \
noto-sans-hebrew-fonts \
noto-sans-imperialaramaic-fonts \
noto-sans-inscriptionalpahlavi-fonts \
noto-sans-inscriptionalparthian-fonts \
noto-sans-javanese-fonts \
noto-sans-jp-black-fonts \
noto-sans-jp-bold-fonts \
noto-sans-jp-demilight-fonts \
noto-sans-jp-fonts \
noto-sans-jp-light-fonts \
noto-sans-jp-medium-fonts \
noto-sans-jp-mono-fonts \
noto-sans-jp-regular-fonts \
noto-sans-jp-thin-fonts \
noto-sans-kaithi-fonts \
noto-sans-kannada-fonts \
noto-sans-kannada-ui-fonts \
noto-sans-kayahli-fonts \
noto-sans-kharoshthi-fonts \
noto-sans-khmer-fonts \
noto-sans-khmer-ui-fonts \
noto-sans-kr-black-fonts \
noto-sans-kr-bold-fonts \
noto-sans-kr-demilight-fonts \
noto-sans-kr-fonts \
noto-sans-kr-light-fonts \
noto-sans-kr-medium-fonts \
noto-sans-kr-mono-fonts \
noto-sans-kr-regular-fonts \
noto-sans-kr-thin-fonts \
noto-sans-lao-fonts \
noto-sans-lao-ui-fonts \
noto-sans-lepcha-fonts \
noto-sans-limbu-fonts \
noto-sans-linearb-fonts \
noto-sans-lisu-fonts \
noto-sans-lycian-fonts \
noto-sans-lydian-fonts \
noto-sans-malayalam-fonts \
noto-sans-malayalam-ui-fonts \
noto-sans-mandaic-fonts \
noto-sans-meeteimayek-fonts \
noto-sans-mongolian-fonts \
noto-sans-mono-fonts \
noto-sans-myanmar-fonts \
noto-sans-myanmar-ui-fonts \
noto-sans-newtailue-fonts \
noto-sans-nko-fonts \
noto-sans-ogham-fonts \
noto-sans-olchiki-fonts \
noto-sans-olditalic-fonts \
noto-sans-oldpersian-fonts \
noto-sans-oldsoutharabian-fonts \
noto-sans-oldturkic-fonts \
noto-sans-oriya-fonts \
noto-sans-oriya-ui-fonts \
noto-sans-osage-fonts \
noto-sans-osmanya-fonts \
noto-sans-phagspa-fonts \
noto-sans-phoenician-fonts \
noto-sans-rejang-fonts \
noto-sans-runic-fonts \
noto-sans-samaritan-fonts \
noto-sans-saurashtra-fonts \
noto-sans-sc-black-fonts \
noto-sans-sc-bold-fonts \
noto-sans-sc-demilight-fonts \
noto-sans-sc-fonts \
noto-sans-sc-light-fonts \
noto-sans-sc-medium-fonts \
noto-sans-sc-mono-fonts \
noto-sans-sc-regular-fonts \
noto-sans-sc-thin-fonts \
noto-sans-shavian-fonts \
noto-sans-sinhala-fonts \
noto-sans-sinhala-ui-fonts \
noto-sans-sundanese-fonts \
noto-sans-sylotinagri-fonts \
noto-sans-symbols-fonts \
noto-sans-symbols2-fonts \
noto-sans-syriaceastern-fonts \
noto-sans-syriacestrangela-fonts \
noto-sans-syriacwestern-fonts \
noto-sans-tagalog-fonts \
noto-sans-tagbanwa-fonts \
noto-sans-taile-fonts \
noto-sans-taitham-fonts \
noto-sans-taiviet-fonts \
noto-sans-tamil-fonts \
noto-sans-tamil-ui-fonts \
noto-sans-tc-black-fonts \
noto-sans-tc-bold-fonts \
noto-sans-tc-demilight-fonts \
noto-sans-tc-fonts \
noto-sans-tc-light-fonts \
noto-sans-tc-medium-fonts \
noto-sans-tc-mono-fonts \
noto-sans-tc-regular-fonts \
noto-sans-tc-thin-fonts \
noto-sans-telugu-fonts \
noto-sans-telugu-ui-fonts \
noto-sans-thaana-fonts \
noto-sans-thai-fonts \
noto-sans-thai-ui-fonts \
noto-sans-tibetan-fonts \
noto-sans-tifinagh-fonts \
noto-sans-ugaritic-fonts \
noto-sans-vai-fonts \
noto-sans-yi-fonts \
noto-serif-armenian-fonts \
noto-serif-bengali-fonts \
noto-serif-devanagari-fonts \
noto-serif-display-fonts \
noto-serif-ethiopic-fonts \
noto-serif-fonts \
noto-serif-georgian-fonts \
noto-serif-gujarati-fonts \
noto-serif-hebrew-fonts \
noto-serif-jp-black-fonts \
noto-serif-jp-bold-fonts \
noto-serif-jp-extralight-fonts \
noto-serif-jp-fonts \
noto-serif-jp-light-fonts \
noto-serif-jp-medium-fonts \
noto-serif-jp-regular-fonts \
noto-serif-jp-semibold-fonts \
noto-serif-kannada-fonts \
noto-serif-khmer-fonts \
noto-serif-kr-black-fonts \
noto-serif-kr-bold-fonts \
noto-serif-kr-extralight-fonts \
noto-serif-kr-fonts \
noto-serif-kr-light-fonts \
noto-serif-kr-medium-fonts \
noto-serif-kr-regular-fonts \
noto-serif-kr-semibold-fonts \
noto-serif-lao-fonts \
noto-serif-malayalam-fonts \
noto-serif-myanmar-fonts \
noto-serif-sc-black-fonts \
noto-serif-sc-bold-fonts \
noto-serif-sc-extralight-fonts \
noto-serif-sc-fonts \
noto-serif-sc-light-fonts \
noto-serif-sc-medium-fonts \
noto-serif-sc-regular-fonts \
noto-serif-sc-semibold-fonts \
noto-serif-sinhala-fonts \
noto-serif-tamil-fonts \
noto-serif-tc-black-fonts \
noto-serif-tc-bold-fonts \
noto-serif-tc-extralight-fonts \
noto-serif-tc-fonts \
noto-serif-tc-light-fonts \
noto-serif-tc-medium-fonts \
noto-serif-tc-regular-fonts \
noto-serif-tc-semibold-fonts \
noto-serif-telugu-fonts \
noto-serif-thai-fonts \
nuosu-fonts \
officecodepro-fonts \
opengost-otf-fonts \
opengost-ttf-fonts \
orkhon-fonts \
paglinawan-quicksand-fonts \
pagul-fonts \
paratype-pt-mono-fonts \
paratype-pt-sans-fonts \
paratype-pt-serif-fonts \
patterns-fonts-fonts \
phetsarath-fonts \
pothana2000-fonts \
powerline-fonts \
raleway-fonts \
rmit-sansforgetica-fonts \
rovasiras-kende-fonts \
rovasiras-roga-fonts \
saja-cascadia-code-fonts \
saweri-fonts \
sazanami-fonts \
sgi-bitmap-fonts \
shannpersand-comicshanns-fonts \
sil-abyssinica-fonts \
sil-andika-fonts \
sil-charis-fonts \
sil-doulos-fonts \
sil-gentium-fonts \
sil-mondulkiri-fonts \
sil-padauk-fonts \
steinberg-bravura-fonts \
stix-fonts \
stix-integrals-fonts \
stix-pua-fonts \
stix-sizes-fonts \
stix-variants-fonts \
sundanese-unicode-fonts \
tagbanwa-fonts \
tai-heritage-pro-fonts \
terminus-bitmap-fonts \
terminus-ttf-fonts \
termsyn-bitmap-fonts \
texlive-academicons-fonts \
texlive-accanthis-fonts \
texlive-adforn-fonts \
texlive-adfsymbols-fonts \
texlive-aesupp-fonts \
texlive-alegreya-fonts \
texlive-alfaslabone-fonts \
texlive-algolrevived-fonts \
texlive-alkalami-fonts \
texlive-allrunes-fonts \
texlive-almendra-fonts \
texlive-almfixed-fonts \
texlive-amiri-fonts \
texlive-amsfonts-fonts \
texlive-anonymouspro-fonts \
texlive-antiqua-fonts \
texlive-antt-fonts \
texlive-arabi-fonts \
texlive-arabtex-fonts \
texlive-aramaic-serto-fonts \
texlive-archaic-fonts \
texlive-archivo-fonts \
texlive-arev-fonts \
texlive-arimo-fonts \
texlive-armtex-fonts \
texlive-arphic-fonts \
texlive-arphic-ttf-fonts \
texlive-arvo-fonts \
texlive-Asana-Math-fonts \
texlive-asapsym-fonts \
texlive-ascii-font-fonts \
texlive-ascmac-fonts \
texlive-aspectratio-fonts \
texlive-atkinson-fonts \
texlive-augie-fonts \
texlive-auncial-new-fonts \
texlive-aurical-fonts \
texlive-avantgar-fonts \
texlive-baekmuk-fonts \
texlive-bartel-chess-fonts \
texlive-baskervald-fonts \
texlive-baskervaldx-fonts \
texlive-baskervillef-fonts \
texlive-bbold-type1-fonts \
texlive-belleek-fonts \
texlive-bera-fonts \
texlive-berenisadf-fonts \
texlive-beuron-fonts \
texlive-bguq-fonts \
texlive-bitter-fonts \
texlive-bookhands-fonts \
texlive-bookman-fonts \
texlive-boondox-fonts \
texlive-brushscr-fonts \
texlive-burmese-fonts \
texlive-cabin-fonts \
texlive-caladea-fonts \
texlive-calligra-type1-fonts \
texlive-cantarell-fonts \
texlive-carlito-fonts \
texlive-carolmin-ps-fonts \
texlive-cascadia-code-fonts \
texlive-cbcoptic-fonts \
texlive-cbfonts-fonts \
texlive-cc-pl-fonts \
texlive-ccicons-fonts \
texlive-charissil-fonts \
texlive-charter-fonts \
texlive-chemarrow-fonts \
texlive-chivo-fonts \
texlive-cinzel-fonts \
texlive-cjhebrew-fonts \
texlive-clara-fonts \
texlive-clearsans-fonts \
texlive-cm-lgc-fonts \
texlive-cm-super-fonts \
texlive-cm-unicode-fonts \
texlive-cmathbb-fonts \
texlive-cmcyr-fonts \
texlive-cmexb-fonts \
texlive-cmll-fonts \
texlive-cmsrb-fonts \
texlive-cmupint-fonts \
texlive-cochineal-fonts \
texlive-coelacanth-fonts \
texlive-comfortaa-fonts \
texlive-comicneue-fonts \
texlive-concmath-fonts \
texlive-context-fonts \
texlive-cormorantgaramond-fonts \
texlive-countriesofeurope-fonts \
texlive-courier-fonts \
texlive-courierten-fonts \
texlive-crimson-fonts \
texlive-crimsonpro-fonts \
texlive-cryst-fonts \
texlive-cs-fonts \
texlive-cuprum-fonts \
texlive-cyklop-fonts \
texlive-dad-fonts \
texlive-dantelogo-fonts \
texlive-dejavu-fonts \
texlive-dictsym-fonts \
texlive-domitian-fonts \
texlive-doublestroke-fonts \
texlive-doulossil-fonts \
texlive-dozenal-fonts \
texlive-drm-fonts \
texlive-droid-fonts \
texlive-dsserif-fonts \
texlive-dutchcal-fonts \
texlive-ebgaramond-fonts \
texlive-eczar-fonts \
texlive-ektype-tanka-fonts \
texlive-electrum-fonts \
texlive-epigrafica-fonts \
texlive-epiolmec-fonts \
texlive-erewhon-fonts \
texlive-erewhon-math-fonts \
texlive-esint-type1-fonts \
texlive-esrelation-fonts \
texlive-esstix-fonts \
texlive-esvect-fonts \
texlive-etbb-fonts \
texlive-ethiop-t1-fonts \
texlive-eurosym-fonts \
texlive-fandol-fonts \
texlive-fbb-fonts \
texlive-fdsymbol-fonts \
texlive-fetamont-fonts \
texlive-fge-fonts \
texlive-figbas-fonts \
texlive-fira-fonts \
texlive-firamath-fonts \
texlive-foekfont-fonts \
texlive-fonetika-fonts \
texlive-fontawesome-fonts \
texlive-fontawesome5-fonts \
texlive-fontmfizz-fonts \
texlive-fonts-churchslavonic-fonts \
texlive-fonts-tlwg-fonts \
texlive-forum-fonts \
texlive-fourier-fonts \
texlive-fpl-fonts \
texlive-frcursive-fonts \
texlive-frederika2016-fonts \
texlive-frimurer-fonts \
texlive-garamond-libre-fonts \
texlive-garamond-math-fonts \
texlive-gentium-tug-fonts \
texlive-gfsartemisia-fonts \
texlive-gfsbaskerville-fonts \
texlive-gfsbodoni-fonts \
texlive-gfscomplutum-fonts \
texlive-gfsdidot-fonts \
texlive-gfsdidotclassic-fonts \
texlive-gfsneohellenic-fonts \
texlive-gfsneohellenicmath-fonts \
texlive-gfsporson-fonts \
texlive-gfssolomos-fonts \
texlive-gillius-fonts \
texlive-gnu-freefont-fonts \
texlive-gofonts-fonts \
texlive-gregoriotex-fonts \
texlive-grotesq-fonts \
texlive-gudea-fonts \
texlive-hacm-fonts \
texlive-haranoaji-extra-fonts \
texlive-haranoaji-fonts \
texlive-helmholtz-ellis-ji-notation-fonts \
texlive-helvetic-fonts \
texlive-heuristica-fonts \
texlive-hfbright-fonts \
texlive-hindmadurai-fonts \
texlive-hmtrump-fonts \
texlive-ibarra-fonts \
texlive-ibygrk-fonts \
texlive-imfellenglish-fonts \
texlive-inconsolata-fonts \
texlive-initials-fonts \
texlive-inriafonts-fonts \
texlive-inter-fonts \
texlive-ipaex-fonts \
texlive-ipaex-type1-fonts \
texlive-iwona-fonts \
texlive-jmn-fonts \
texlive-josefin-fonts \
texlive-junicode-fonts \
texlive-kerkis-fonts \
texlive-knitting-fonts \
texlive-kpfonts-fonts \
texlive-kpfonts-otf-fonts \
texlive-kurier-fonts \
texlive-latex-fonts \
texlive-lato-fonts \
texlive-lexend-fonts \
texlive-libertine-fonts \
texlive-libertinus-fonts \
texlive-libertinus-fonts-fonts \
texlive-libertinus-type1-fonts \
texlive-libertinust1math-fonts \
texlive-librebaskerville-fonts \
texlive-librebodoni-fonts \
texlive-librecaslon-fonts \
texlive-librefranklin-fonts \
texlive-libris-fonts \
texlive-lilyglyphs-fonts \
texlive-linearA-fonts \
texlive-linguisticspro-fonts \
texlive-lm-fonts \
texlive-lm-math-fonts \
texlive-lobster2-fonts \
texlive-logix-fonts \
texlive-lxfonts-fonts \
texlive-magra-fonts \
texlive-manfnt-font-fonts \
texlive-marcellus-fonts \
texlive-marvosym-fonts \
texlive-mathabx-type1-fonts \
texlive-mathdesign-fonts \
texlive-mathpazo-fonts \
texlive-mdsymbol-fonts \
texlive-merriweather-fonts \
texlive-metapost-fonts \
texlive-mflogo-font-fonts \
texlive-miama-fonts \
texlive-mintspirit-fonts \
texlive-missaali-fonts \
texlive-mlmodern-fonts \
texlive-mnsymbol-fonts \
texlive-montex-fonts \
texlive-montserrat-fonts \
texlive-musixtex-fonts \
texlive-musixtex-fonts-fonts \
texlive-mxedruli-fonts \
texlive-nanumtype1-fonts \
texlive-ncntrsbk-fonts \
texlive-newcomputermodern-fonts \
texlive-newpx-fonts \
texlive-newtx-fonts \
texlive-newtxsf-fonts \
texlive-newtxtt-fonts \
texlive-niceframe-type1-fonts \
texlive-nimbus15-fonts \
texlive-noto-emoji-fonts \
texlive-noto-fonts \
texlive-notomath-fonts \
texlive-novel-fonts \
texlive-nunito-fonts \
texlive-ocherokee-fonts \
texlive-ocr-b-outline-fonts \
texlive-oinuit-fonts \
texlive-old-arrows-fonts \
texlive-oldstandard-fonts \
texlive-omega-fonts \
texlive-opensans-fonts \
texlive-oswald-fonts \
texlive-overlock-fonts \
texlive-padauk-fonts \
texlive-palatino-fonts \
texlive-paratype-fonts \
texlive-pdftex-fonts \
texlive-phaistos-fonts \
texlive-philokalia-fonts \
texlive-pigpen-fonts \
texlive-pl-fonts \
texlive-playfair-fonts \
texlive-plex-fonts \
texlive-plimsoll-fonts \
texlive-poiretone-fonts \
texlive-poltawski-fonts \
texlive-prodint-fonts \
texlive-ptex-fonts \
texlive-punknova-fonts \
texlive-pxfonts-fonts \
texlive-qualitype-fonts \
texlive-quattrocento-fonts \
texlive-raleway-fonts \
texlive-recycle-fonts \
texlive-roboto-fonts \
texlive-rojud-fonts \
texlive-romande-fonts \
texlive-rosario-fonts \
texlive-rsfs-fonts \
texlive-sanskrit-t1-fonts \
texlive-sansmathfonts-fonts \
texlive-scanpages-fonts \
texlive-scholax-fonts \
texlive-semaphor-fonts \
texlive-shobhika-fonts \
texlive-skaknew-fonts \
texlive-sourcecodepro-fonts \
texlive-sourcesanspro-fonts \
texlive-sourceserifpro-fonts \
texlive-spectral-fonts \
texlive-starfont-fonts \
texlive-staves-fonts \
texlive-step-fonts \
texlive-stepgreek-fonts \
texlive-stickstoo-fonts \
texlive-stix-fonts \
texlive-stix2-otf-fonts \
texlive-stix2-type1-fonts \
texlive-stmaryrd-fonts \
texlive-superiors-fonts \
texlive-svrsymbols-fonts \
texlive-symbol-fonts \
texlive-tabvar-fonts \
texlive-tapir-fonts \
texlive-tempora-fonts \
texlive-tex-gyre-fonts \
texlive-tex-gyre-math-fonts \
texlive-tfrupee-fonts \
texlive-theanodidot-fonts \
texlive-theanomodern-fonts \
texlive-theanooldstyle-fonts \
texlive-times-fonts \
texlive-tinos-fonts \
texlive-tipa-fonts \
texlive-trajan-fonts \
texlive-twemoji-colr-fonts \
texlive-txfonts-fonts \
texlive-txfontsb-fonts \
texlive-txuprcal-fonts \
texlive-typicons-fonts \
texlive-uhc-fonts \
texlive-umtypewriter-fonts \
texlive-unfonts-core-fonts \
texlive-unfonts-extra-fonts \
texlive-universalis-fonts \
texlive-uptex-fonts \
texlive-utopia-fonts \
texlive-velthuis-fonts \
texlive-venturisadf-fonts \
texlive-vntex-fonts \
texlive-wadalab-fonts \
texlive-wasy-type1-fonts \
texlive-xcharter-fonts \
texlive-xits-fonts \
texlive-xypic-fonts \
texlive-yfonts-t1-fonts \
texlive-yhmath-fonts \
texlive-yinit-otf-fonts \
texlive-zapfchan-fonts \
texlive-zapfding-fonts \
thai-fonts \
thessalonica-oldstandard-otf-fonts \
thessalonica-oldstandard-ttf-fonts \
thessalonica-tempora-lgc-otf-fonts \
thessalonica-tempora-lgc-ttf-fonts \
thessalonica-theano-otf-fonts \
thessalonica-theano-ttf-fonts \
thryomanes-fonts \
tibetan-machine-uni-fonts \
tiro-bangla-fonts \
tiro-devahindi-fonts \
tiro-devamarathi-fonts \
tiro-devasanskrit-fonts \
tiro-gurmukhi-fonts \
tiro-indigo-fonts \
tiro-kannada-fonts \
tiro-tamil-fonts \
tiro-telugu-fonts \
tuladha-jejeg-fonts \
tv-fonts \
ubuntu-fonts \
un-fonts \
unifraktur-fonts \
vlgothic-fonts \
vollkorn-fonts \
wang-fonts \
wqy-bitmap-fonts \
wqy-microhei-fonts \
wqy-zenhei-fonts \
x11-japanese-bitmap-fonts \
xano-mincho-fonts \
xorg-x11-fonts