From 18065d27d3ef6cc6c0624f56f74575f588f13580 Mon Sep 17 00:00:00 2001
From: Janne Mareike Koschinski <janne@kuschku.de>
Date: Mon, 26 Jul 2021 01:02:45 +0200
Subject: [PATCH] Significant cleanup and improvements
---
api/album_get.go | 2 +-
api/album_list.go | 2 +-
api/albumimage_get.go | 2 +-
api/albumimage_list.go | 2 +-
api/image_get.go | 2 +-
api/image_list.go | 2 +-
api/image_upload.go | 40 ++++++++++
auth/parse_user.go | 2 +-
cmd/backend/main.go | 6 +-
cmd/frontend/main.go | 7 +-
configuration/auth.go | 2 +-
configuration/backend.go | 9 +++
configuration/configuration.go | 9 ---
configuration/conversion.go | 14 ++--
configuration/database.go | 4 +-
configuration/frontend.go | 10 +++
configuration/queue.go | 8 +-
configuration/redis.go | 4 +-
configuration/storage.go | 10 +++
environment/backend.go | 57 +++++++++++++++
environment/{environment.go => frontend.go} | 49 +++----------
go.mod | 9 ++-
go.sum | 78 +++++++++++++++++++-
s3/upload_source.go | 19 -----
storage/storage.go | 55 ++++++++++++++
task/image_resize.go | 56 +-------------
task/image_resize_processor.go | 81 +++++++++++++++++++++
util/return_json.go | 13 +++-
28 files changed, 397 insertions(+), 157 deletions(-)
create mode 100644 api/image_upload.go
create mode 100644 configuration/backend.go
delete mode 100644 configuration/configuration.go
create mode 100644 configuration/frontend.go
create mode 100644 configuration/storage.go
create mode 100644 environment/backend.go
rename environment/{environment.go => frontend.go} (52%)
delete mode 100644 s3/upload_source.go
create mode 100644 storage/storage.go
create mode 100644 task/image_resize_processor.go
diff --git a/api/album_get.go b/api/album_get.go
index 741a40a..063b8e0 100644
--- a/api/album_get.go
+++ b/api/album_get.go
@@ -8,7 +8,7 @@ import (
"net/http"
)
-func GetAlbum(env environment.Environment) http.Handler {
+func GetAlbum(env environment.FrontendEnvironment) http.Handler {
return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
vars := mux.Vars(request)
album, err := env.Repositories.Albums.Get(vars["albumId"])
diff --git a/api/album_list.go b/api/album_list.go
index 9429759..c29898b 100644
--- a/api/album_list.go
+++ b/api/album_list.go
@@ -8,7 +8,7 @@ import (
"net/http"
)
-func ListAlbums(env environment.Environment) http.Handler {
+func ListAlbums(env environment.FrontendEnvironment) http.Handler {
return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
user, err := auth.ParseUser(request, env)
if err != nil {
diff --git a/api/albumimage_get.go b/api/albumimage_get.go
index f140856..f85a592 100644
--- a/api/albumimage_get.go
+++ b/api/albumimage_get.go
@@ -8,7 +8,7 @@ import (
"net/http"
)
-func GetAlbumImage(env environment.Environment) http.Handler {
+func GetAlbumImage(env environment.FrontendEnvironment) http.Handler {
return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
vars := mux.Vars(request)
albumImage, err := env.Repositories.AlbumImages.Get(vars["albumId"], vars["imageId"])
diff --git a/api/albumimage_list.go b/api/albumimage_list.go
index 5b284df..9782852 100644
--- a/api/albumimage_list.go
+++ b/api/albumimage_list.go
@@ -8,7 +8,7 @@ import (
"net/http"
)
-func ListAlbumImages(env environment.Environment) http.Handler {
+func ListAlbumImages(env environment.FrontendEnvironment) http.Handler {
return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
vars := mux.Vars(request)
albumImages, err := env.Repositories.AlbumImages.List(vars["albumId"])
diff --git a/api/image_get.go b/api/image_get.go
index 9ec538a..b18a010 100644
--- a/api/image_get.go
+++ b/api/image_get.go
@@ -8,7 +8,7 @@ import (
"net/http"
)
-func GetImage(env environment.Environment) http.Handler {
+func GetImage(env environment.FrontendEnvironment) http.Handler {
return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
vars := mux.Vars(request)
image, err := env.Repositories.Images.Get(vars["imageId"])
diff --git a/api/image_list.go b/api/image_list.go
index 4148b4c..712219b 100644
--- a/api/image_list.go
+++ b/api/image_list.go
@@ -8,7 +8,7 @@ import (
"net/http"
)
-func ListImages(env environment.Environment) http.Handler {
+func ListImages(env environment.FrontendEnvironment) http.Handler {
return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
user, err := auth.ParseUser(request, env)
if err != nil {
diff --git a/api/image_upload.go b/api/image_upload.go
new file mode 100644
index 0000000..e1b8e5f
--- /dev/null
+++ b/api/image_upload.go
@@ -0,0 +1,40 @@
+package api
+
+import (
+ "context"
+ "git.kuschku.de/justjanne/imghost-frontend/auth"
+ "git.kuschku.de/justjanne/imghost-frontend/environment"
+ "git.kuschku.de/justjanne/imghost-frontend/model"
+ "git.kuschku.de/justjanne/imghost-frontend/util"
+ "net/http"
+)
+
+func UploadImage(env environment.FrontendEnvironment) http.Handler {
+ return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
+ user, err := auth.ParseUser(request, env)
+ if err != nil {
+ http.Error(writer, err.Error(), http.StatusUnauthorized)
+ return
+ }
+
+ var image model.Image
+ image.Id = "testid"
+ image.Owner = user.Id
+ err = env.Repositories.Images.Create(image)
+ if err != nil {
+ http.Error(writer, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ err = env.Storage.Upload(
+ context.Background(),
+ env.Configuration.Storage.ConversionBucket,
+ image.Id,
+ request.Body)
+ if err != nil {
+ http.Error(writer, err.Error(), http.StatusInternalServerError)
+ return
+ }
+
+ util.ReturnJson(writer, image)
+ })
+}
diff --git a/auth/parse_user.go b/auth/parse_user.go
index fab59f3..2de4deb 100644
--- a/auth/parse_user.go
+++ b/auth/parse_user.go
@@ -6,7 +6,7 @@ import (
"net/http"
)
-func ParseUser(request *http.Request, env environment.Environment) (user model.User, err error) {
+func ParseUser(request *http.Request, env environment.FrontendEnvironment) (user model.User, err error) {
// TODO: Implement actual user auth
user = model.User{
Id: "ad45284c-be4d-4546-8171-41cf126ac091",
diff --git a/cmd/backend/main.go b/cmd/backend/main.go
index 102b6d2..6b14815 100644
--- a/cmd/backend/main.go
+++ b/cmd/backend/main.go
@@ -12,7 +12,7 @@ import (
)
func main() {
- var config configuration.Configuration
+ var config configuration.BackendConfiguration
configFile, err := os.Open("config.yaml")
if err != nil {
panic(err)
@@ -22,14 +22,14 @@ func main() {
panic(err)
}
- env, err := environment.NewServerEnvironment(config)
+ env, err := environment.NewBackendEnvironment(config)
if err != nil {
panic(err)
}
defer env.Destroy()
mux := asynq.NewServeMux()
- mux.HandleFunc(config.Conversion.ResizeTaskId, task.HandleImageResizeTask)
+ mux.Handle(config.Conversion.TaskId, task.NewImageProcessor(env))
if err := env.QueueServer.Run(mux); err != nil {
log.Fatalf("could not run server: %v", err)
}
diff --git a/cmd/frontend/main.go b/cmd/frontend/main.go
index d101b77..1b15115 100644
--- a/cmd/frontend/main.go
+++ b/cmd/frontend/main.go
@@ -13,7 +13,7 @@ import (
)
func main() {
- var config configuration.Configuration
+ var config configuration.FrontendConfiguration
configFile, err := os.Open("config.yaml")
if err != nil {
panic(err)
@@ -23,7 +23,7 @@ func main() {
panic(err)
}
- env, err := environment.NewClientEnvironment(config)
+ env, err := environment.NewFrontendEnvironment(config)
if err != nil {
panic(err)
}
@@ -34,6 +34,9 @@ func main() {
router.Handle(
"/api/v1/images",
api.ListImages(env)).Methods(http.MethodGet)
+ router.Handle(
+ "/api/v1/images",
+ api.UploadImage(env)).Methods(http.MethodPost)
router.Handle(
"/api/v1/images/{imageId}",
api.GetImage(env)).Methods(http.MethodGet)
diff --git a/configuration/auth.go b/configuration/auth.go
index d87ba42..3ec8900 100644
--- a/configuration/auth.go
+++ b/configuration/auth.go
@@ -1,5 +1,5 @@
package configuration
type AuthConfiguration struct {
- RolePrefix string `json:"role_prefix"`
+ RolePrefix string `json:"role_prefix" yaml:"role-prefix"`
}
diff --git a/configuration/backend.go b/configuration/backend.go
new file mode 100644
index 0000000..6b14fbf
--- /dev/null
+++ b/configuration/backend.go
@@ -0,0 +1,9 @@
+package configuration
+
+type BackendConfiguration struct {
+ Queue QueueConfiguration `json:"queue" yaml:"queue"`
+ Database DatabaseConfiguration `json:"database" yaml:"database"`
+ Redis RedisConfiguration `json:"redis" yaml:"redis"`
+ Conversion ConversionConfiguration `json:"conversion" yaml:"conversion"`
+ Storage StorageConfiguration `json:"storage" yaml:"storage"`
+}
diff --git a/configuration/configuration.go b/configuration/configuration.go
deleted file mode 100644
index 42558ae..0000000
--- a/configuration/configuration.go
+++ /dev/null
@@ -1,9 +0,0 @@
-package configuration
-
-type Configuration struct {
- Queue QueueConfiguration `json:"queue"`
- Database DatabaseConfiguration `json:"database"`
- Redis RedisConfiguration `json:"redis"`
- Conversion ConversionConfiguration `json:"conversion"`
- Auth AuthConfiguration `json:"auth"`
-}
diff --git a/configuration/conversion.go b/configuration/conversion.go
index 11d896d..00c1ab9 100644
--- a/configuration/conversion.go
+++ b/configuration/conversion.go
@@ -6,11 +6,11 @@ import (
)
type ConversionConfiguration struct {
- TaskId string `json:"task_id"`
- MaxRetry int `json:"max_retry"`
- Timeout types.Timeout `json:"timeout"`
- Queue string `json:"queue"`
- UniqueTimeout types.Timeout `json:"unique_timeout"`
- Quality imgconv.Quality `json:"quality"`
- Sizes []imgconv.Size `json:"sizes"`
+ TaskId string `json:"task_id" yaml:"task-id"`
+ MaxRetry int `json:"max_retry" yaml:"max-retry"`
+ Timeout types.Timeout `json:"timeout" yaml:"timeout"`
+ Queue string `json:"queue" yaml:"queue"`
+ UniqueTimeout types.Timeout `json:"unique_timeout" yaml:"unique-timeout"`
+ Quality imgconv.Quality `json:"quality" yaml:"quality"`
+ Sizes []imgconv.Size `json:"sizes" yaml:"sizes"`
}
diff --git a/configuration/database.go b/configuration/database.go
index 2719add..0741d7a 100644
--- a/configuration/database.go
+++ b/configuration/database.go
@@ -1,6 +1,6 @@
package configuration
type DatabaseConfiguration struct {
- Type string `json:"type"`
- Url string `json:"url"`
+ Type string `json:"type" yaml:"type"`
+ Url string `json:"url" yaml:"url"`
}
diff --git a/configuration/frontend.go b/configuration/frontend.go
new file mode 100644
index 0000000..a4675ad
--- /dev/null
+++ b/configuration/frontend.go
@@ -0,0 +1,10 @@
+package configuration
+
+type FrontendConfiguration struct {
+ Queue QueueConfiguration `json:"queue" yaml:"queue"`
+ Database DatabaseConfiguration `json:"database" yaml:"database"`
+ Redis RedisConfiguration `json:"redis" yaml:"redis"`
+ Conversion ConversionConfiguration `json:"conversion" yaml:"conversion"`
+ Storage StorageConfiguration `json:"storage" yaml:"storage"`
+ Auth AuthConfiguration `json:"auth" yaml:"auth"`
+}
diff --git a/configuration/queue.go b/configuration/queue.go
index b10105c..7edee6d 100644
--- a/configuration/queue.go
+++ b/configuration/queue.go
@@ -3,8 +3,8 @@ package configuration
import "git.kuschku.de/justjanne/imghost-frontend/configuration/types"
type QueueConfiguration struct {
- Concurrency int `json:"concurrency"`
- LogLevel types.Severity `json:"log_level"`
- StrictPriority bool `json:"strict_priority"`
- Queues map[string]int `json:"queues"`
+ Concurrency int `json:"concurrency" yaml:"concurrency"`
+ LogLevel types.Severity `json:"log_level" yaml:"log-level"`
+ StrictPriority bool `json:"strict_priority" yaml:"strict-priority"`
+ Queues map[string]int `json:"queues" yaml:"queues"`
}
diff --git a/configuration/redis.go b/configuration/redis.go
index cb93818..aff3eeb 100644
--- a/configuration/redis.go
+++ b/configuration/redis.go
@@ -1,6 +1,6 @@
package configuration
type RedisConfiguration struct {
- Address string `json:"address"`
- Password string `json:"password"`
+ Address string `json:"address" yaml:"address"`
+ Password string `json:"password" yaml:"password"`
}
diff --git a/configuration/storage.go b/configuration/storage.go
new file mode 100644
index 0000000..5158a35
--- /dev/null
+++ b/configuration/storage.go
@@ -0,0 +1,10 @@
+package configuration
+
+type StorageConfiguration struct {
+ Endpoint string `json:"endpoint" yaml:"endpoint"`
+ Secure bool `json:"secure" yaml:"secure"`
+ ImageBucket string `json:"image_bucket" yaml:"image-bucket"`
+ ConversionBucket string `json:"conversion_bucket" yaml:"conversion-bucket"`
+ AccessKey string `json:"access_key" yaml:"access-key"`
+ SecretKey string `json:"secret_key" yaml:"secret-key"`
+}
diff --git a/environment/backend.go b/environment/backend.go
new file mode 100644
index 0000000..73fd20a
--- /dev/null
+++ b/environment/backend.go
@@ -0,0 +1,57 @@
+package environment
+
+import (
+ "git.kuschku.de/justjanne/imghost-frontend/configuration"
+ "git.kuschku.de/justjanne/imghost-frontend/repo"
+ "git.kuschku.de/justjanne/imghost-frontend/storage"
+ "github.com/hibiken/asynq"
+ "github.com/jmoiron/sqlx"
+)
+
+type BackendEnvironment struct {
+ Configuration configuration.BackendConfiguration
+ QueueServer *asynq.Server
+ Database *sqlx.DB
+ Repositories Repositories
+ Storage storage.Storage
+}
+
+func NewBackendEnvironment(config configuration.BackendConfiguration) (env BackendEnvironment, err error) {
+ env.Configuration = config
+ if env.Database, err = sqlx.Open(config.Database.Type, config.Database.Url); err != nil {
+ return
+ }
+ if env.Repositories.Images, err = repo.NewImageRepo(env.Database); err != nil {
+ return
+ }
+ if env.Repositories.Albums, err = repo.NewAlbumRepo(env.Database); err != nil {
+ return
+ }
+ if env.Repositories.AlbumImages, err = repo.NewAlbumImageRepo(env.Database); err != nil {
+ return
+ }
+ if env.Storage, err = storage.NewStorage(env.Configuration.Storage); err != nil {
+ return
+ }
+ env.QueueServer = asynq.NewServer(
+ asynq.RedisClientOpt{
+ Addr: config.Redis.Address,
+ Password: config.Redis.Password,
+ },
+ asynq.Config{
+ Concurrency: config.Queue.Concurrency,
+ LogLevel: asynq.LogLevel(config.Queue.LogLevel),
+ Queues: config.Queue.Queues,
+ StrictPriority: config.Queue.StrictPriority,
+ },
+ )
+ return env, err
+}
+
+func (env BackendEnvironment) Destroy() error {
+ if err := env.Database.Close(); err != nil {
+ return err
+ }
+ env.QueueServer.Shutdown()
+ return nil
+}
diff --git a/environment/environment.go b/environment/frontend.go
similarity index 52%
rename from environment/environment.go
rename to environment/frontend.go
index 31a0e4e..b05e301 100644
--- a/environment/environment.go
+++ b/environment/frontend.go
@@ -3,20 +3,21 @@ package environment
import (
"git.kuschku.de/justjanne/imghost-frontend/configuration"
"git.kuschku.de/justjanne/imghost-frontend/repo"
+ "git.kuschku.de/justjanne/imghost-frontend/storage"
"github.com/hibiken/asynq"
"github.com/jmoiron/sqlx"
"time"
)
-type Environment struct {
- Configuration configuration.Configuration
+type FrontendEnvironment struct {
+ Configuration configuration.FrontendConfiguration
QueueClient *asynq.Client
- QueueServer *asynq.Server
Database *sqlx.DB
Repositories Repositories
+ Storage storage.Storage
}
-func newCommonEnvironment(config configuration.Configuration) (env Environment, err error) {
+func NewFrontendEnvironment(config configuration.FrontendConfiguration) (env FrontendEnvironment, err error) {
env.Configuration = config
if env.Database, err = sqlx.Open(config.Database.Type, config.Database.Url); err != nil {
return
@@ -30,13 +31,8 @@ func newCommonEnvironment(config configuration.Configuration) (env Environment,
if env.Repositories.AlbumImages, err = repo.NewAlbumImageRepo(env.Database); err != nil {
return
}
- return
-}
-
-func NewClientEnvironment(config configuration.Configuration) (Environment, error) {
- env, err := newCommonEnvironment(config)
- if err != nil {
- return env, err
+ if env.Storage, err = storage.NewStorage(env.Configuration.Storage); err != nil {
+ return
}
env.QueueClient = asynq.NewClient(asynq.RedisClientOpt{
Addr: config.Redis.Address,
@@ -52,37 +48,12 @@ func NewClientEnvironment(config configuration.Configuration) (Environment, erro
return env, err
}
-func NewServerEnvironment(config configuration.Configuration) (Environment, error) {
- env, err := newCommonEnvironment(config)
- if err != nil {
- return env, err
- }
- env.QueueServer = asynq.NewServer(
- asynq.RedisClientOpt{
- Addr: config.Redis.Address,
- Password: config.Redis.Password,
- },
- asynq.Config{
- Concurrency: config.Queue.Concurrency,
- LogLevel: asynq.LogLevel(config.Queue.LogLevel),
- Queues: config.Queue.Queues,
- StrictPriority: config.Queue.StrictPriority,
- },
- )
- return env, err
-}
-
-func (env Environment) Destroy() error {
+func (env FrontendEnvironment) Destroy() error {
if err := env.Database.Close(); err != nil {
return err
}
- if env.QueueClient != nil {
- if err := env.QueueClient.Close(); err != nil {
- return err
- }
- }
- if env.QueueServer != nil {
- env.QueueServer.Shutdown()
+ if err := env.QueueClient.Close(); err != nil {
+ return err
}
return nil
}
diff --git a/go.mod b/go.mod
index 3698c09..f3dfb02 100644
--- a/go.mod
+++ b/go.mod
@@ -3,14 +3,15 @@ module git.kuschku.de/justjanne/imghost-frontend
go 1.13
require (
- github.com/gorilla/mux v1.8.0 // indirect
+ github.com/gorilla/mux v1.8.0
github.com/hibiken/asynq v0.18.2
github.com/jmoiron/sqlx v1.3.4
- github.com/justjanne/imgconv v1.0.3 // indirect
+ github.com/justjanne/imgconv v1.2.0
github.com/lib/pq v1.10.2
- github.com/stretchr/testify v1.4.0 // indirect
+ github.com/minio/minio-go/v7 v7.0.12
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 // indirect
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect
golang.org/x/text v0.3.6 // indirect
- gopkg.in/yaml.v2 v2.2.8 // indirect
+ gopkg.in/gographics/imagick.v2 v2.6.0
+ gopkg.in/yaml.v2 v2.2.8
)
diff --git a/go.sum b/go.sum
index 89853b5..a3ffe7e 100644
--- a/go.sum
+++ b/go.sum
@@ -1,11 +1,19 @@
+cloud.google.com/go v0.26.0 h1:e0WKqKTd5BnrG8aKH3J3h+QvEIQtSUcf2n5UZ5ZgLtQ=
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473 h1:4cmBvAEBNJaGARUEs3/suWRyfyBfhf7I60WBZq+bv2w=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
@@ -13,7 +21,9 @@ github.com/go-redis/redis/v7 v7.4.0 h1:7obg6wUoj05T0EpY0o8B59S9w5yeMWql7sw2kwNW1
github.com/go-redis/redis/v7 v7.4.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg=
github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/mock v1.1.1 h1:G5FRp8JnTd7RQH5kemVNlMeyXQAztQ3mOWV95KxsXH8=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -30,8 +40,13 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/hibiken/asynq v0.18.2 h1:hbLVygnmQMc2evTmNildtUWaUPHIBtggA59MoaHp/bY=
@@ -40,10 +55,20 @@ github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/jmoiron/sqlx v1.3.4 h1:wv+0IJZfL5z0uZoUjlpKgHkgaFSYD+r9CfrXjEXsO7w=
github.com/jmoiron/sqlx v1.3.4/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ=
-github.com/justjanne/imgconv v1.0.3 h1:XXgqeLJ1ibV0XCdosMqVV65CdpmDADTnC3yBR2w52vE=
-github.com/justjanne/imgconv v1.0.3/go.mod h1:8VxkQjMdEOziT9LJ8Mrqtz7SbmqlpExl69K8/CZWeAc=
+github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
+github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/justjanne/imgconv v1.1.0 h1:0QdU9zV65BCO/SG/5UnjSkG4hD2Fp7XISdmnDPC1O4o=
+github.com/justjanne/imgconv v1.1.0/go.mod h1:8VxkQjMdEOziT9LJ8Mrqtz7SbmqlpExl69K8/CZWeAc=
+github.com/justjanne/imgconv v1.2.0 h1:gkrflWEvT2rCMzeyvm7Bm65+XdnmI/Bm2l9r7ngFG5A=
+github.com/justjanne/imgconv v1.2.0/go.mod h1:8VxkQjMdEOziT9LJ8Mrqtz7SbmqlpExl69K8/CZWeAc=
+github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s=
+github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
@@ -52,6 +77,20 @@ github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8=
github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg=
github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
+github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4=
+github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
+github.com/minio/minio-go/v7 v7.0.12 h1:/4pxUdwn9w0QEryNkrrWaodIESPRX+NxpO0Q6hVdaAA=
+github.com/minio/minio-go/v7 v7.0.12/go.mod h1:S23iSP5/gbMwtxeY5FM71R+TkAYyzEdoNEDDwpt8yWs=
+github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
+github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo=
github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -59,42 +98,68 @@ github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
+github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
+github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
+github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
+github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
+github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
go.uber.org/goleak v0.10.0 h1:G3eWbSNIskeRqtsN/1uI5B+eP73y3JUuBsv9AZjehb4=
go.uber.org/goleak v0.10.0/go.mod h1:VCZuO8V8mFPlL0F5J5GK1rtHV3DrFcQ1R8ryq7FK0aI=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f h1:aZp0e2vLN4MToVqnjNEYEtrEA8RH8U8FN1CU7JgqsPU=
+golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4 h1:c2HOrn5iMezYjSlGPncknSEr/8x5LELb/ilJbXi9DEA=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 h1:XQyxROzUlZH+WIQwySDgnISgOivlhjIEwaQaJEJrrN0=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
@@ -107,16 +172,21 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135 h1:5Beo0mZN8dRzgrMMkDp0jc8YXQKx9DiJ2k1dkvGsn5A=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.27.0 h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
@@ -128,12 +198,15 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/gographics/imagick.v2 v2.6.0 h1:ewRsUQk3QkjGumERlndbFn/kTYRjyMaPY5gxwpuAhik=
gopkg.in/gographics/imagick.v2 v2.6.0/go.mod h1:/QVPLV/iKdNttRKthmDkeeGg+vdHurVEPc8zkU0XgBk=
+gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww=
+gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
@@ -143,4 +216,5 @@ gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc h1:/hemPrYIhOhy8zYrNj+069zDB68us2sMGsfkFJO0iZs=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
diff --git a/s3/upload_source.go b/s3/upload_source.go
deleted file mode 100644
index 8393c17..0000000
--- a/s3/upload_source.go
+++ /dev/null
@@ -1,19 +0,0 @@
-package s3
-
-import (
- "git.kuschku.de/justjanne/imghost-frontend/environment"
- "github.com/justjanne/imgconv"
-)
-
-// TODO: Implement
-func UploadSource(env environment.Environment, imageId string, source string) error {
- return nil
-}
-
-func DownloadSource(env environment.Environment, imageId string) (string, error) {
- return "", nil
-}
-
-func UploadImage(env environment.Environment, imageId string, format imgconv.Size, source string) error {
- return nil
-}
diff --git a/storage/storage.go b/storage/storage.go
new file mode 100644
index 0000000..8a4065d
--- /dev/null
+++ b/storage/storage.go
@@ -0,0 +1,55 @@
+package storage
+
+import (
+ "context"
+ "git.kuschku.de/justjanne/imghost-frontend/configuration"
+ "github.com/minio/minio-go/v7"
+ "github.com/minio/minio-go/v7/pkg/credentials"
+ "io"
+ "os"
+)
+
+type Storage struct {
+ config configuration.StorageConfiguration
+ s3client *minio.Client
+}
+
+func NewStorage(config configuration.StorageConfiguration) (storage Storage, err error) {
+ storage.config = config
+ storage.s3client, err = minio.New(config.Endpoint, &minio.Options{
+ Creds: credentials.NewStaticV4(config.AccessKey, config.SecretKey, ""),
+ Secure: config.Secure,
+ })
+ return
+}
+
+func (storage Storage) UploadFile(ctx context.Context, bucketName string, fileName string, file *os.File) (err error) {
+ _, err = storage.s3client.FPutObject(
+ ctx,
+ bucketName,
+ fileName,
+ file.Name(),
+ minio.PutObjectOptions{})
+ return
+}
+
+func (storage Storage) Upload(ctx context.Context, bucketName string, fileName string, reader io.ReadCloser) (err error) {
+ _, err = storage.s3client.PutObject(
+ ctx,
+ bucketName,
+ fileName,
+ reader,
+ -1,
+ minio.PutObjectOptions{})
+ return
+}
+
+func (storage Storage) DownloadFile(ctx context.Context, bucketName string, fileName string, file *os.File) (err error) {
+ err = storage.s3client.FGetObject(
+ ctx,
+ bucketName,
+ fileName,
+ file.Name(),
+ minio.GetObjectOptions{})
+ return
+}
diff --git a/task/image_resize.go b/task/image_resize.go
index 99021cc..28d5e28 100644
--- a/task/image_resize.go
+++ b/task/image_resize.go
@@ -1,15 +1,10 @@
package task
import (
- "context"
"encoding/json"
"git.kuschku.de/justjanne/imghost-frontend/configuration"
- "git.kuschku.de/justjanne/imghost-frontend/environment"
- "git.kuschku.de/justjanne/imghost-frontend/s3"
- "git.kuschku.de/justjanne/imghost-frontend/util"
"github.com/hibiken/asynq"
"github.com/justjanne/imgconv"
- "gopkg.in/gographics/imagick.v2/imagick"
)
type ImageResizePayload struct {
@@ -18,7 +13,7 @@ type ImageResizePayload struct {
Quality imgconv.Quality
}
-func NewResizeTask(imageId string, config configuration.Configuration) (task *asynq.Task, err error) {
+func NewResizeTask(imageId string, config configuration.FrontendConfiguration) (task *asynq.Task, err error) {
payload, err := json.Marshal(ImageResizePayload{
ImageId: imageId,
Sizes: config.Conversion.Sizes,
@@ -27,53 +22,6 @@ func NewResizeTask(imageId string, config configuration.Configuration) (task *as
if err != nil {
return
}
- task = asynq.NewTask(config.Conversion.ResizeTaskId, payload)
- return
-}
-
-func HandleImageResizeTask(ctx context.Context, task *asynq.Task) (err error) {
- // TODO: Handle environment for tasks
- env := environment.Environment{}
-
- var payload ImageResizePayload
- if err = json.Unmarshal(task.Payload(), &payload); err != nil {
- return
- }
-
- wand := imagick.NewMagickWand()
- defer wand.Destroy()
-
- file, err := s3.DownloadSource(env, payload.ImageId)
- if err != nil {
- return
- }
- if err = wand.ReadImage(file); err != nil {
- return
- }
- var originalImage imgconv.ImageHandle
- if originalImage, err = imgconv.NewImage(wand); err != nil {
- return err
- }
-
- err = util.LaunchGoroutines(len(payload.Sizes), func(index int) error {
- size := payload.Sizes[index]
- // TODO: Allocate temp file
- tmpFile := ""
- image := originalImage.CloneImage()
- if err := image.Crop(size); err != nil {
- return err
- }
- if err := image.Resize(size); err != nil {
- return err
- }
- if err := image.Write(payload.Quality, tmpFile); err != nil {
- return err
- }
- if err := s3.UploadImage(env, payload.ImageId, size, tmpFile); err != nil {
- return err
- }
- return nil
- })
-
+ task = asynq.NewTask(config.Conversion.TaskId, payload)
return
}
diff --git a/task/image_resize_processor.go b/task/image_resize_processor.go
new file mode 100644
index 0000000..a3071b4
--- /dev/null
+++ b/task/image_resize_processor.go
@@ -0,0 +1,81 @@
+package task
+
+import (
+ "context"
+ "encoding/json"
+ "git.kuschku.de/justjanne/imghost-frontend/environment"
+ "git.kuschku.de/justjanne/imghost-frontend/util"
+ "github.com/hibiken/asynq"
+ "github.com/justjanne/imgconv"
+ "gopkg.in/gographics/imagick.v2/imagick"
+ "io/ioutil"
+)
+
+type ImageProcessor struct {
+ env environment.BackendEnvironment
+}
+
+func NewImageProcessor(env environment.BackendEnvironment) *ImageProcessor {
+ return &ImageProcessor{
+ env,
+ }
+}
+
+func (processor *ImageProcessor) ProcessTask(ctx context.Context, task *asynq.Task) (err error) {
+ var payload ImageResizePayload
+ if err = json.Unmarshal(task.Payload(), &payload); err != nil {
+ return
+ }
+
+ wand := imagick.NewMagickWand()
+ defer wand.Destroy()
+
+ sourceFile, err := ioutil.TempFile("", payload.ImageId)
+ if err != nil {
+ return
+ }
+ err = processor.env.Storage.DownloadFile(
+ ctx,
+ processor.env.Configuration.Storage.ConversionBucket,
+ payload.ImageId,
+ sourceFile)
+ if err != nil {
+ return
+ }
+ if err = wand.ReadImageFile(sourceFile); err != nil {
+ return
+ }
+ var originalImage imgconv.ImageHandle
+ if originalImage, err = imgconv.NewImage(wand); err != nil {
+ return err
+ }
+
+ err = util.LaunchGoroutines(len(payload.Sizes), func(index int) error {
+ outputFile, err := ioutil.TempFile("", payload.ImageId)
+ if err != nil {
+ return err
+ }
+
+ size := payload.Sizes[index]
+ image := originalImage.CloneImage()
+ if err := image.Crop(size); err != nil {
+ return err
+ }
+ if err := image.Resize(size); err != nil {
+ return err
+ }
+ if err := image.Write(payload.Quality, outputFile); err != nil {
+ return err
+ }
+ if err := processor.env.Storage.UploadFile(
+ ctx,
+ processor.env.Configuration.Storage.ImageBucket,
+ payload.ImageId,
+ outputFile); err != nil {
+ return err
+ }
+ return nil
+ })
+
+ return
+}
diff --git a/util/return_json.go b/util/return_json.go
index aad85ef..bd7c820 100644
--- a/util/return_json.go
+++ b/util/return_json.go
@@ -7,7 +7,16 @@ import (
func ReturnJson(writer http.ResponseWriter, data interface{}) {
writer.Header().Set("Content-Type", "application/json")
- if err := json.NewEncoder(writer).Encode(data); err != nil {
- writer.WriteHeader(http.StatusInternalServerError)
+ bytes, err := json.Marshal(&data)
+ if err != nil {
+ http.Error(writer, err.Error(), http.StatusInternalServerError)
+ return
+ } else {
+ writer.WriteHeader(200)
+ _, err := writer.Write(bytes)
+ if err != nil {
+ http.Error(writer, err.Error(), http.StatusInternalServerError)
+ return
+ }
}
}
--
GitLab