46
internal/auth/actor.go
Normal file
46
internal/auth/actor.go
Normal file
@@ -0,0 +1,46 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"git.apinb.com/bsm-sdk/core/middleware"
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
const (
|
||||
ActorTypeUser = "user"
|
||||
ActorTypeService = "service"
|
||||
|
||||
actorContextKey = "files.auth.actor"
|
||||
)
|
||||
|
||||
type Actor struct {
|
||||
Type string
|
||||
ID uint
|
||||
Identity string
|
||||
}
|
||||
|
||||
func FromContext(ctx *gin.Context) (Actor, bool) {
|
||||
if actor, ok := ctx.Get(actorContextKey); ok {
|
||||
serviceActor, ok := actor.(Actor)
|
||||
if ok && serviceActor.Type == ActorTypeService && serviceActor.ID == 0 && serviceActor.Identity != "" {
|
||||
return serviceActor, true
|
||||
}
|
||||
}
|
||||
|
||||
claims, err := middleware.ParseAuth(ctx)
|
||||
if err != nil || claims == nil || claims.ID == 0 || claims.Identity == "" {
|
||||
return Actor{}, false
|
||||
}
|
||||
|
||||
return Actor{
|
||||
Type: ActorTypeUser,
|
||||
ID: claims.ID,
|
||||
Identity: claims.Identity,
|
||||
}, true
|
||||
}
|
||||
|
||||
func withActor(ctx *gin.Context, actor Actor) {
|
||||
if actor.Type != ActorTypeService || actor.ID != 0 || actor.Identity == "" {
|
||||
return
|
||||
}
|
||||
ctx.Set(actorContextKey, actor)
|
||||
}
|
||||
31
internal/auth/service.go
Normal file
31
internal/auth/service.go
Normal file
@@ -0,0 +1,31 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"crypto/subtle"
|
||||
"log"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"git.apinb.com/ops/files/internal/config"
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
func ServiceAuth() gin.HandlerFunc {
|
||||
return func(ctx *gin.Context) {
|
||||
serviceName := strings.TrimSpace(ctx.GetHeader("Service-Name"))
|
||||
secretKey := strings.TrimSpace(ctx.GetHeader("Secret-Key"))
|
||||
serviceSecret, exists := config.Spec.ServiceClients[serviceName]
|
||||
if serviceName == "" || secretKey == "" || !exists || strings.TrimSpace(serviceSecret) == "" || subtle.ConstantTimeCompare([]byte(serviceSecret), []byte(secretKey)) != 1 {
|
||||
log.Printf("服务鉴权失败: service=%q", serviceName)
|
||||
ctx.JSON(http.StatusUnauthorized, gin.H{"error": "Unauthorized"})
|
||||
ctx.Abort()
|
||||
return
|
||||
}
|
||||
|
||||
withActor(ctx, Actor{
|
||||
Type: ActorTypeService,
|
||||
Identity: serviceName,
|
||||
})
|
||||
ctx.Next()
|
||||
}
|
||||
}
|
||||
147
internal/config/config.go
Normal file
147
internal/config/config.go
Normal file
@@ -0,0 +1,147 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"log"
|
||||
"net"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.apinb.com/bsm-sdk/core/conf"
|
||||
)
|
||||
|
||||
var (
|
||||
Spec SrvConfig
|
||||
configNameRegexp = regexp.MustCompile(`^[a-z0-9-]+$`)
|
||||
)
|
||||
|
||||
type SrvConfig struct {
|
||||
conf.Base `yaml:",inline"`
|
||||
Databases *conf.DBConf `yaml:"Databases"`
|
||||
MicroService *conf.MicroServiceConf `yaml:"MicroService"`
|
||||
Etcd *conf.EtcdConf `yaml:"Etcd"`
|
||||
ObjectStorage ObjectStorageConf `yaml:"ObjectStorage"`
|
||||
Namespaces map[string]NamespaceConf `yaml:"Namespaces"`
|
||||
ServiceClients map[string]string `yaml:"ServiceClients"`
|
||||
Cleanup CleanupConf `yaml:"Cleanup"`
|
||||
}
|
||||
|
||||
type ObjectStorageConf struct {
|
||||
Provider string `yaml:"Provider"`
|
||||
Endpoint string `yaml:"Endpoint"`
|
||||
Region string `yaml:"Region"`
|
||||
Bucket string `yaml:"Bucket"`
|
||||
PublicBaseURL string `yaml:"PublicBaseURL"`
|
||||
AccessKeyID string `yaml:"AccessKeyID"`
|
||||
AccessKeySecret string `yaml:"AccessKeySecret"`
|
||||
PresignTTLSeconds int64 `yaml:"PresignTTLSeconds"`
|
||||
}
|
||||
|
||||
type NamespaceConf struct {
|
||||
Prefix string `yaml:"Prefix"`
|
||||
MaxSizeMB int64 `yaml:"MaxSizeMB"`
|
||||
AllowedExtensions []string `yaml:"AllowedExtensions"`
|
||||
}
|
||||
|
||||
type CleanupConf struct {
|
||||
IntervalSeconds int64 `yaml:"IntervalSeconds"`
|
||||
}
|
||||
|
||||
func New(srvKey string) {
|
||||
conf.New(srvKey, &Spec)
|
||||
|
||||
Spec.Port = conf.CheckPort(Spec.Port)
|
||||
Spec.BindIP = conf.CheckIP(Spec.BindIP)
|
||||
Spec.Addr = net.JoinHostPort(Spec.BindIP, Spec.Port)
|
||||
|
||||
validate()
|
||||
conf.PrintInfo(Spec.Addr)
|
||||
}
|
||||
|
||||
func validate() {
|
||||
validateRequired("Service", Spec.Service)
|
||||
validateRequired("Cache", Spec.Cache)
|
||||
|
||||
if Spec.ObjectStorage.Provider != "aliyun" {
|
||||
configError("ObjectStorage.Provider", "必须为 aliyun")
|
||||
}
|
||||
|
||||
validateRequired("ObjectStorage.Endpoint", Spec.ObjectStorage.Endpoint)
|
||||
validateRequired("ObjectStorage.Region", Spec.ObjectStorage.Region)
|
||||
validateRequired("ObjectStorage.Bucket", Spec.ObjectStorage.Bucket)
|
||||
validateRequired("ObjectStorage.PublicBaseURL", Spec.ObjectStorage.PublicBaseURL)
|
||||
validateRequired("ObjectStorage.AccessKeyID", Spec.ObjectStorage.AccessKeyID)
|
||||
validateRequired("ObjectStorage.AccessKeySecret", Spec.ObjectStorage.AccessKeySecret)
|
||||
|
||||
if Spec.ObjectStorage.PresignTTLSeconds < 60 || Spec.ObjectStorage.PresignTTLSeconds > 3600 {
|
||||
configError("ObjectStorage.PresignTTLSeconds", "必须在 60 到 3600 秒之间")
|
||||
}
|
||||
|
||||
if len(Spec.Namespaces) == 0 {
|
||||
configError("Namespaces", "不能为空")
|
||||
}
|
||||
for name, namespace := range Spec.Namespaces {
|
||||
path := "Namespaces." + name
|
||||
if !configNameRegexp.MatchString(name) {
|
||||
configError(path, "名称仅允许小写字母、数字和连字符")
|
||||
}
|
||||
if !isSafeRelativeObjectPath(namespace.Prefix) {
|
||||
configError(path+".Prefix", "必须是安全的相对对象路径")
|
||||
}
|
||||
if namespace.MaxSizeMB <= 0 {
|
||||
configError(path+".MaxSizeMB", "必须大于 0")
|
||||
}
|
||||
if len(namespace.AllowedExtensions) == 0 {
|
||||
configError(path+".AllowedExtensions", "不能为空")
|
||||
}
|
||||
for index, extension := range namespace.AllowedExtensions {
|
||||
extensionPath := path + ".AllowedExtensions[" + stringIndex(index) + "]"
|
||||
if !strings.HasPrefix(extension, ".") {
|
||||
configError(extensionPath, "必须以 . 开头")
|
||||
}
|
||||
namespace.AllowedExtensions[index] = strings.ToLower(extension)
|
||||
}
|
||||
Spec.Namespaces[name] = namespace
|
||||
}
|
||||
|
||||
if len(Spec.ServiceClients) == 0 {
|
||||
configError("ServiceClients", "不能为空")
|
||||
}
|
||||
for name, secret := range Spec.ServiceClients {
|
||||
path := "ServiceClients." + name
|
||||
if !configNameRegexp.MatchString(name) {
|
||||
configError(path, "服务名仅允许小写字母、数字和连字符")
|
||||
}
|
||||
validateRequired(path, secret)
|
||||
}
|
||||
|
||||
if Spec.Cleanup.IntervalSeconds <= 0 {
|
||||
configError("Cleanup.IntervalSeconds", "必须大于 0")
|
||||
}
|
||||
}
|
||||
|
||||
func validateRequired(path, value string) {
|
||||
if strings.TrimSpace(value) == "" {
|
||||
configError(path, "不能为空")
|
||||
}
|
||||
}
|
||||
|
||||
func isSafeRelativeObjectPath(path string) bool {
|
||||
if path == "" || strings.HasPrefix(path, "/") || strings.Contains(path, "\\") {
|
||||
return false
|
||||
}
|
||||
for _, segment := range strings.Split(path, "/") {
|
||||
if segment == "." || segment == ".." {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func configError(path, message string) {
|
||||
log.Fatalf("配置项 %s 无效:%s", path, message)
|
||||
}
|
||||
|
||||
func stringIndex(index int) string {
|
||||
return strconv.Itoa(index)
|
||||
}
|
||||
17
internal/errors/errors.go
Normal file
17
internal/errors/errors.go
Normal file
@@ -0,0 +1,17 @@
|
||||
package errors
|
||||
|
||||
import "git.apinb.com/bsm-sdk/core/errcode"
|
||||
|
||||
var (
|
||||
ErrInvalidParameter = errcode.NewError(2101, "参数错误")
|
||||
ErrNamespaceNotAllowed = errcode.NewError(2102, "命名空间不允许")
|
||||
ErrExtensionNotAllowed = errcode.NewError(2103, "扩展名不允许")
|
||||
ErrInvalidFileSize = errcode.NewError(2104, "文件大小不合法")
|
||||
ErrFileNotFound = errcode.NewError(2105, "文件不存在")
|
||||
ErrUploadStatusConflict = errcode.NewError(2106, "上传状态冲突")
|
||||
ErrUploadExpired = errcode.NewError(2107, "上传已过期")
|
||||
ErrObjectInfoMismatch = errcode.NewError(2108, "OSS 对象信息不匹配")
|
||||
ErrUnauthorizedOperation = errcode.NewError(2109, "无权操作")
|
||||
ErrObjectStorageOperation = errcode.NewError(2110, "OSS 操作失败")
|
||||
ErrDatabaseOperation = errcode.NewError(2111, "数据库操作失败")
|
||||
)
|
||||
34
internal/impl/impl.go
Normal file
34
internal/impl/impl.go
Normal file
@@ -0,0 +1,34 @@
|
||||
package impl
|
||||
|
||||
import (
|
||||
"git.apinb.com/bsm-sdk/core/cache/redis"
|
||||
"git.apinb.com/bsm-sdk/core/logger"
|
||||
"git.apinb.com/bsm-sdk/core/with"
|
||||
"git.apinb.com/ops/files/internal/config"
|
||||
"git.apinb.com/ops/files/internal/storage"
|
||||
clientv3 "go.etcd.io/etcd/client/v3"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
var (
|
||||
RedisService *redis.RedisClient
|
||||
EtcdService *clientv3.Client
|
||||
DBService *gorm.DB
|
||||
StorageService *storage.Client
|
||||
Logger *logger.Logger
|
||||
)
|
||||
|
||||
// NewImpl 初始化各类服务实例。
|
||||
func NewImpl() {
|
||||
DBService = with.Databases(config.Spec.Databases, nil)
|
||||
|
||||
var err error
|
||||
StorageService, err = storage.New(config.Spec.ObjectStorage)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
RedisService = with.RedisCache(config.Spec.Cache)
|
||||
EtcdService = with.Etcd(config.Spec.Etcd)
|
||||
logger.New(nil)
|
||||
}
|
||||
79
internal/jobs/pending_upload.go
Normal file
79
internal/jobs/pending_upload.go
Normal file
@@ -0,0 +1,79 @@
|
||||
package jobs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"git.apinb.com/bsm-sdk/core/logger"
|
||||
"git.apinb.com/ops/files/internal/config"
|
||||
"git.apinb.com/ops/files/internal/impl"
|
||||
"git.apinb.com/ops/files/internal/lifecycle"
|
||||
"git.apinb.com/ops/files/internal/models"
|
||||
)
|
||||
|
||||
func StartPendingUploadCleanup() {
|
||||
go func() {
|
||||
cleanupPendingUploads()
|
||||
|
||||
ticker := time.NewTicker(time.Duration(config.Spec.Cleanup.IntervalSeconds) * time.Second)
|
||||
defer ticker.Stop()
|
||||
for range ticker.C {
|
||||
cleanupPendingUploads()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func cleanupPendingUploads() {
|
||||
var fileObjects []models.FileObject
|
||||
now := time.Now()
|
||||
result := impl.DBService.
|
||||
Where(
|
||||
"(status = ? AND expires_at <= ?) OR (status = ? AND (delete_lease_until IS NULL OR delete_lease_until <= ?))",
|
||||
models.FileStatusPending,
|
||||
now,
|
||||
models.FileStatusDeleting,
|
||||
now,
|
||||
).
|
||||
Order("id ASC").
|
||||
Limit(100).
|
||||
Find(&fileObjects)
|
||||
if result.Error != nil {
|
||||
logger.Error("stage=scan")
|
||||
return
|
||||
}
|
||||
|
||||
for _, fileObject := range fileObjects {
|
||||
var lease lifecycle.DeletionLease
|
||||
var claimed bool
|
||||
var err error
|
||||
now = time.Now()
|
||||
if fileObject.Status == models.FileStatusPending {
|
||||
lease, claimed, err = lifecycle.ClaimExpiredPendingDeletion(impl.DBService, fileObject.ID, now)
|
||||
} else {
|
||||
lease, claimed, err = lifecycle.ClaimDeletionRetry(impl.DBService, fileObject.ID, now)
|
||||
}
|
||||
if err != nil {
|
||||
logger.Errorf("identity=%s stage=claim", fileObject.Identity)
|
||||
continue
|
||||
}
|
||||
if !claimed {
|
||||
continue
|
||||
}
|
||||
|
||||
cleanupClaimedFile(fileObject, lease)
|
||||
}
|
||||
}
|
||||
|
||||
func cleanupClaimedFile(fileObject models.FileObject, lease lifecycle.DeletionLease) {
|
||||
operationCtx, cancel := context.WithTimeout(context.Background(), lifecycle.DeleteOperationTimeout)
|
||||
defer cancel()
|
||||
|
||||
if err := impl.StorageService.Delete(operationCtx, fileObject.ObjectKey); err != nil {
|
||||
logger.Errorf("identity=%s stage=oss_delete", fileObject.Identity)
|
||||
return
|
||||
}
|
||||
|
||||
if err := lifecycle.FinalizeDeletion(impl.DBService.WithContext(operationCtx), fileObject.ID, lease.Token); err != nil {
|
||||
logger.Errorf("identity=%s stage=finalize", fileObject.Identity)
|
||||
}
|
||||
}
|
||||
89
internal/lifecycle/deletion.go
Normal file
89
internal/lifecycle/deletion.go
Normal file
@@ -0,0 +1,89 @@
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"git.apinb.com/bsm-sdk/core/utils"
|
||||
"git.apinb.com/ops/files/internal/models"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
const (
|
||||
deletionLeaseDuration = 5 * time.Minute
|
||||
DeleteOperationTimeout = 30 * time.Second
|
||||
)
|
||||
|
||||
type DeletionLease struct {
|
||||
Token string
|
||||
LeaseUntil time.Time
|
||||
}
|
||||
|
||||
func ClaimDeletion(db *gorm.DB, id uint, allowedStatuses []models.FileStatus, now time.Time) (DeletionLease, bool, error) {
|
||||
query := db.Model(&models.FileObject{}).
|
||||
Where("id = ? AND status IN ?", id, allowedStatuses)
|
||||
return claimDeletion(query, now)
|
||||
}
|
||||
|
||||
func ClaimExpiredPendingDeletion(db *gorm.DB, id uint, now time.Time) (DeletionLease, bool, error) {
|
||||
query := db.Model(&models.FileObject{}).
|
||||
Where("id = ? AND status = ? AND expires_at <= ?", id, models.FileStatusPending, now)
|
||||
return claimDeletion(query, now)
|
||||
}
|
||||
|
||||
func ClaimDeletionRetry(db *gorm.DB, id uint, now time.Time) (DeletionLease, bool, error) {
|
||||
query := db.Model(&models.FileObject{}).
|
||||
Where("id = ? AND status = ?", id, models.FileStatusDeleting).
|
||||
Where("delete_lease_until IS NULL OR delete_lease_until <= ?", now)
|
||||
return claimDeletion(query, now)
|
||||
}
|
||||
|
||||
func FinalizeDeletion(db *gorm.DB, id uint, token string) error {
|
||||
if token == "" {
|
||||
return gorm.ErrRecordNotFound
|
||||
}
|
||||
|
||||
return db.Transaction(func(tx *gorm.DB) error {
|
||||
result := tx.Model(&models.FileObject{}).
|
||||
Where("id = ? AND status = ? AND delete_token = ?", id, models.FileStatusDeleting, token).
|
||||
Updates(map[string]any{
|
||||
"status": models.FileStatusExpired,
|
||||
"delete_token": "",
|
||||
"delete_lease_until": nil,
|
||||
})
|
||||
if result.Error != nil {
|
||||
return result.Error
|
||||
}
|
||||
if result.RowsAffected != 1 {
|
||||
return gorm.ErrRecordNotFound
|
||||
}
|
||||
|
||||
result = tx.Where("id = ? AND status = ?", id, models.FileStatusExpired).
|
||||
Delete(&models.FileObject{})
|
||||
if result.Error != nil {
|
||||
return result.Error
|
||||
}
|
||||
if result.RowsAffected != 1 {
|
||||
return gorm.ErrRecordNotFound
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func claimDeletion(query *gorm.DB, now time.Time) (DeletionLease, bool, error) {
|
||||
lease := DeletionLease{
|
||||
Token: utils.ULID(),
|
||||
LeaseUntil: now.Add(deletionLeaseDuration),
|
||||
}
|
||||
result := query.Updates(map[string]any{
|
||||
"status": models.FileStatusDeleting,
|
||||
"delete_token": lease.Token,
|
||||
"delete_lease_until": lease.LeaseUntil,
|
||||
})
|
||||
if result.Error != nil {
|
||||
return DeletionLease{}, false, result.Error
|
||||
}
|
||||
if result.RowsAffected != 1 {
|
||||
return DeletionLease{}, false, nil
|
||||
}
|
||||
return lease, true, nil
|
||||
}
|
||||
61
internal/logic/files/complete.go
Normal file
61
internal/logic/files/complete.go
Normal file
@@ -0,0 +1,61 @@
|
||||
package files
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"git.apinb.com/bsm-sdk/core/infra"
|
||||
fileerrors "git.apinb.com/ops/files/internal/errors"
|
||||
"git.apinb.com/ops/files/internal/impl"
|
||||
"git.apinb.com/ops/files/internal/models"
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
func CompleteUpload(ctx *gin.Context) {
|
||||
fileObject, ok := loadOwnedFile(ctx)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if fileObject.Status != models.FileStatusPending {
|
||||
infra.Response.Error(ctx, fileerrors.ErrUploadStatusConflict)
|
||||
return
|
||||
}
|
||||
|
||||
if fileObject.ExpiresAt == nil || !fileObject.ExpiresAt.After(time.Now()) {
|
||||
infra.Response.Error(ctx, fileerrors.ErrUploadExpired)
|
||||
return
|
||||
}
|
||||
|
||||
objectInfo, err := impl.StorageService.Stat(ctx.Request.Context(), fileObject.ObjectKey)
|
||||
if err != nil {
|
||||
infra.Response.Error(ctx, fileerrors.ErrObjectStorageOperation)
|
||||
return
|
||||
}
|
||||
if objectInfo.Size != fileObject.ExpectedSize {
|
||||
infra.Response.Error(ctx, fileerrors.ErrObjectInfoMismatch)
|
||||
return
|
||||
}
|
||||
|
||||
completedAt := time.Now()
|
||||
result := impl.DBService.Model(&models.FileObject{}).
|
||||
Where("id = ? AND status = ? AND expires_at > ?", fileObject.ID, models.FileStatusPending, completedAt).
|
||||
Updates(map[string]any{
|
||||
"status": models.FileStatusReady,
|
||||
"actual_size": objectInfo.Size,
|
||||
"etag": objectInfo.ETag,
|
||||
"completed_at": completedAt,
|
||||
})
|
||||
if result.Error != nil {
|
||||
infra.Response.Error(ctx, fileerrors.ErrDatabaseOperation)
|
||||
return
|
||||
}
|
||||
if result.RowsAffected != 1 {
|
||||
infra.Response.Error(ctx, fileerrors.ErrUploadStatusConflict)
|
||||
return
|
||||
}
|
||||
|
||||
fileObject.Status = models.FileStatusReady
|
||||
fileObject.ActualSize = objectInfo.Size
|
||||
fileObject.ETag = objectInfo.ETag
|
||||
fileObject.CompletedAt = &completedAt
|
||||
infra.Response.Success(ctx, newFileResponse(fileObject))
|
||||
}
|
||||
50
internal/logic/files/delete.go
Normal file
50
internal/logic/files/delete.go
Normal file
@@ -0,0 +1,50 @@
|
||||
package files
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"git.apinb.com/bsm-sdk/core/infra"
|
||||
fileerrors "git.apinb.com/ops/files/internal/errors"
|
||||
"git.apinb.com/ops/files/internal/impl"
|
||||
"git.apinb.com/ops/files/internal/lifecycle"
|
||||
"git.apinb.com/ops/files/internal/models"
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
func Delete(ctx *gin.Context) {
|
||||
fileObject, ok := loadOwnedFile(ctx)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
deletableStatuses := []models.FileStatus{
|
||||
models.FileStatusPending,
|
||||
models.FileStatusReady,
|
||||
models.FileStatusExpired,
|
||||
}
|
||||
lease, claimed, err := lifecycle.ClaimDeletion(impl.DBService, fileObject.ID, deletableStatuses, time.Now())
|
||||
if err != nil {
|
||||
infra.Response.Error(ctx, fileerrors.ErrDatabaseOperation)
|
||||
return
|
||||
}
|
||||
if !claimed {
|
||||
infra.Response.Error(ctx, fileerrors.ErrUploadStatusConflict)
|
||||
return
|
||||
}
|
||||
|
||||
operationCtx, cancel := context.WithTimeout(ctx.Request.Context(), lifecycle.DeleteOperationTimeout)
|
||||
defer cancel()
|
||||
|
||||
if err := impl.StorageService.Delete(operationCtx, fileObject.ObjectKey); err != nil {
|
||||
infra.Response.Error(ctx, fileerrors.ErrObjectStorageOperation)
|
||||
return
|
||||
}
|
||||
|
||||
if err := lifecycle.FinalizeDeletion(impl.DBService.WithContext(operationCtx), fileObject.ID, lease.Token); err != nil {
|
||||
infra.Response.Error(ctx, fileerrors.ErrDatabaseOperation)
|
||||
return
|
||||
}
|
||||
|
||||
infra.Response.Success(ctx, nil)
|
||||
}
|
||||
68
internal/logic/files/detail.go
Normal file
68
internal/logic/files/detail.go
Normal file
@@ -0,0 +1,68 @@
|
||||
package files
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
|
||||
"git.apinb.com/bsm-sdk/core/infra"
|
||||
"git.apinb.com/ops/files/internal/auth"
|
||||
fileerrors "git.apinb.com/ops/files/internal/errors"
|
||||
"git.apinb.com/ops/files/internal/impl"
|
||||
"git.apinb.com/ops/files/internal/models"
|
||||
"github.com/gin-gonic/gin"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func Detail(ctx *gin.Context) {
|
||||
fileObject, ok := loadOwnedFile(ctx)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if fileObject.Status != models.FileStatusReady {
|
||||
infra.Response.Error(ctx, fileerrors.ErrFileNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
infra.Response.Success(ctx, newFileResponse(fileObject))
|
||||
}
|
||||
|
||||
func loadOwnedFile(ctx *gin.Context) (models.FileObject, bool) {
|
||||
actor, ok := auth.FromContext(ctx)
|
||||
if !ok {
|
||||
infra.Response.Error(ctx, fileerrors.ErrUnauthorizedOperation)
|
||||
return models.FileObject{}, false
|
||||
}
|
||||
|
||||
identity := strings.TrimSpace(ctx.Param("identity"))
|
||||
if identity == "" {
|
||||
infra.Response.Error(ctx, fileerrors.ErrInvalidParameter)
|
||||
return models.FileObject{}, false
|
||||
}
|
||||
|
||||
var fileObject models.FileObject
|
||||
if err := impl.DBService.
|
||||
Where("identity = ? AND owner_type = ? AND owner_identity = ?", identity, actor.Type, actor.Identity).
|
||||
First(&fileObject).Error; err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
infra.Response.Error(ctx, fileerrors.ErrFileNotFound)
|
||||
} else {
|
||||
infra.Response.Error(ctx, fileerrors.ErrDatabaseOperation)
|
||||
}
|
||||
return models.FileObject{}, false
|
||||
}
|
||||
|
||||
return fileObject, true
|
||||
}
|
||||
|
||||
func newFileResponse(fileObject models.FileObject) FileResponse {
|
||||
return FileResponse{
|
||||
FileID: fileObject.Identity,
|
||||
ObjectKey: fileObject.ObjectKey,
|
||||
URL: impl.StorageService.PublicURL(fileObject.ObjectKey),
|
||||
Filename: fileObject.OriginalName,
|
||||
Size: fileObject.ActualSize,
|
||||
ContentType: fileObject.ContentType,
|
||||
ETag: fileObject.ETag,
|
||||
Status: fileObject.Status,
|
||||
}
|
||||
}
|
||||
139
internal/logic/files/init.go
Normal file
139
internal/logic/files/init.go
Normal file
@@ -0,0 +1,139 @@
|
||||
package files
|
||||
|
||||
import (
|
||||
"math"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"git.apinb.com/bsm-sdk/core/infra"
|
||||
"git.apinb.com/bsm-sdk/core/utils"
|
||||
"git.apinb.com/ops/files/internal/auth"
|
||||
"git.apinb.com/ops/files/internal/config"
|
||||
fileerrors "git.apinb.com/ops/files/internal/errors"
|
||||
"git.apinb.com/ops/files/internal/impl"
|
||||
"git.apinb.com/ops/files/internal/models"
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
const bytesPerMB int64 = 1024 * 1024
|
||||
|
||||
func InitUpload(ctx *gin.Context) {
|
||||
var request InitUploadRequest
|
||||
if err := ctx.ShouldBindJSON(&request); err != nil {
|
||||
infra.Response.Error(ctx, fileerrors.ErrInvalidParameter)
|
||||
return
|
||||
}
|
||||
|
||||
actor, ok := auth.FromContext(ctx)
|
||||
if !ok {
|
||||
infra.Response.Error(ctx, fileerrors.ErrUnauthorizedOperation)
|
||||
return
|
||||
}
|
||||
|
||||
namespace, namespaceConfig, err := validateNamespace(request.Namespace)
|
||||
if err != nil {
|
||||
infra.Response.Error(ctx, err)
|
||||
return
|
||||
}
|
||||
|
||||
filename, extension, err := validateFilename(request.Filename, namespaceConfig)
|
||||
if err != nil {
|
||||
infra.Response.Error(ctx, err)
|
||||
return
|
||||
}
|
||||
if utf8.RuneCountInString(filename) > 255 {
|
||||
infra.Response.Error(ctx, fileerrors.ErrInvalidParameter)
|
||||
return
|
||||
}
|
||||
|
||||
if !validFileSize(request.Size, namespaceConfig.MaxSizeMB) {
|
||||
infra.Response.Error(ctx, fileerrors.ErrInvalidFileSize)
|
||||
return
|
||||
}
|
||||
|
||||
contentType := strings.TrimSpace(request.ContentType)
|
||||
if contentType == "" || utf8.RuneCountInString(contentType) > 255 {
|
||||
infra.Response.Error(ctx, fileerrors.ErrInvalidParameter)
|
||||
return
|
||||
}
|
||||
|
||||
fileID := utils.ULID()
|
||||
objectKey := buildObjectKey(namespaceConfig.Prefix, time.Now(), fileID, extension)
|
||||
upload, err := impl.StorageService.PresignPut(ctx.Request.Context(), objectKey, contentType)
|
||||
if err != nil {
|
||||
infra.Response.Error(ctx, fileerrors.ErrObjectStorageOperation)
|
||||
return
|
||||
}
|
||||
|
||||
fileObject := models.FileObject{
|
||||
Identity: fileID,
|
||||
Namespace: namespace,
|
||||
Provider: config.Spec.ObjectStorage.Provider,
|
||||
Bucket: impl.StorageService.Bucket(),
|
||||
ObjectKey: objectKey,
|
||||
OriginalName: filename,
|
||||
Extension: extension,
|
||||
ContentType: contentType,
|
||||
ExpectedSize: request.Size,
|
||||
Status: models.FileStatusPending,
|
||||
OwnerType: actor.Type,
|
||||
OwnerID: actor.ID,
|
||||
OwnerIdentity: actor.Identity,
|
||||
ExpiresAt: &upload.ExpiresAt,
|
||||
}
|
||||
if err := impl.DBService.Create(&fileObject).Error; err != nil {
|
||||
infra.Response.Error(ctx, fileerrors.ErrDatabaseOperation)
|
||||
return
|
||||
}
|
||||
|
||||
infra.Response.Success(ctx, InitUploadResponse{
|
||||
FileID: fileID,
|
||||
ObjectKey: objectKey,
|
||||
Upload: upload,
|
||||
})
|
||||
}
|
||||
|
||||
func validateNamespace(value string) (string, config.NamespaceConf, error) {
|
||||
namespace := strings.TrimSpace(value)
|
||||
namespaceConfig, ok := config.Spec.Namespaces[namespace]
|
||||
if !ok {
|
||||
return "", config.NamespaceConf{}, fileerrors.ErrNamespaceNotAllowed
|
||||
}
|
||||
return namespace, namespaceConfig, nil
|
||||
}
|
||||
|
||||
func validateFilename(value string, namespaceConfig config.NamespaceConf) (string, string, error) {
|
||||
filename := strings.TrimSpace(value)
|
||||
if strings.ContainsRune(filename, 0) {
|
||||
return "", "", fileerrors.ErrInvalidParameter
|
||||
}
|
||||
|
||||
filename = filepath.Base(filename)
|
||||
if filename == "" || filename == "." || filename == ".." {
|
||||
return "", "", fileerrors.ErrInvalidParameter
|
||||
}
|
||||
|
||||
extension := strings.ToLower(filepath.Ext(filename))
|
||||
if extension == "" || !containsExtension(namespaceConfig.AllowedExtensions, extension) {
|
||||
return "", "", fileerrors.ErrExtensionNotAllowed
|
||||
}
|
||||
return filename, extension, nil
|
||||
}
|
||||
|
||||
func containsExtension(extensions []string, extension string) bool {
|
||||
for _, allowedExtension := range extensions {
|
||||
if allowedExtension == extension {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func validFileSize(size, maxSizeMB int64) bool {
|
||||
if size < 1 || maxSizeMB < 1 || maxSizeMB > math.MaxInt64/bytesPerMB {
|
||||
return false
|
||||
}
|
||||
return size <= maxSizeMB*bytesPerMB
|
||||
}
|
||||
10
internal/logic/files/object_key.go
Normal file
10
internal/logic/files/object_key.go
Normal file
@@ -0,0 +1,10 @@
|
||||
package files
|
||||
|
||||
import (
|
||||
"path"
|
||||
"time"
|
||||
)
|
||||
|
||||
func buildObjectKey(prefix string, now time.Time, fileID, extension string) string {
|
||||
return path.Join(prefix, now.Format("2006"), now.Format("01"), fileID+extension)
|
||||
}
|
||||
30
internal/logic/files/types.go
Normal file
30
internal/logic/files/types.go
Normal file
@@ -0,0 +1,30 @@
|
||||
package files
|
||||
|
||||
import (
|
||||
"git.apinb.com/ops/files/internal/models"
|
||||
"git.apinb.com/ops/files/internal/storage"
|
||||
)
|
||||
|
||||
type InitUploadRequest struct {
|
||||
Namespace string `json:"namespace" binding:"required"`
|
||||
Filename string `json:"filename" binding:"required"`
|
||||
Size int64 `json:"size" binding:"required"`
|
||||
ContentType string `json:"content_type" binding:"required"`
|
||||
}
|
||||
|
||||
type InitUploadResponse struct {
|
||||
FileID string `json:"file_id"`
|
||||
ObjectKey string `json:"object_key"`
|
||||
Upload storage.UploadInstruction `json:"upload"`
|
||||
}
|
||||
|
||||
type FileResponse struct {
|
||||
FileID string `json:"file_id"`
|
||||
ObjectKey string `json:"object_key"`
|
||||
URL string `json:"url"`
|
||||
Filename string `json:"filename"`
|
||||
Size int64 `json:"size"`
|
||||
ContentType string `json:"content_type"`
|
||||
ETag string `json:"etag"`
|
||||
Status models.FileStatus `json:"status"`
|
||||
}
|
||||
11
internal/logic/ping/hello.go
Normal file
11
internal/logic/ping/hello.go
Normal file
@@ -0,0 +1,11 @@
|
||||
package ping
|
||||
|
||||
import (
|
||||
"git.apinb.com/bsm-sdk/core/infra"
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
func Hello(ctx *gin.Context) {
|
||||
infra.Response.Success(ctx, "Files Service is running!")
|
||||
return
|
||||
}
|
||||
46
internal/models/file_object.go
Normal file
46
internal/models/file_object.go
Normal file
@@ -0,0 +1,46 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
type FileStatus string
|
||||
|
||||
const (
|
||||
FileStatusPending FileStatus = "pending"
|
||||
FileStatusReady FileStatus = "ready"
|
||||
FileStatusDeleting FileStatus = "deleting"
|
||||
FileStatusExpired FileStatus = "expired"
|
||||
)
|
||||
|
||||
type FileObject struct {
|
||||
ID uint `gorm:"primaryKey" json:"id"`
|
||||
Identity string `gorm:"size:64;not null;uniqueIndex" json:"identity"`
|
||||
Namespace string `gorm:"size:64;not null" json:"namespace"`
|
||||
Provider string `gorm:"size:32;not null" json:"provider"`
|
||||
Bucket string `gorm:"size:255;not null" json:"bucket"`
|
||||
ObjectKey string `gorm:"size:512;not null;uniqueIndex" json:"object_key"`
|
||||
OriginalName string `gorm:"size:255;not null" json:"original_name"`
|
||||
Extension string `gorm:"size:32" json:"extension"`
|
||||
ContentType string `gorm:"size:255" json:"content_type"`
|
||||
ExpectedSize int64 `gorm:"not null" json:"expected_size"`
|
||||
ActualSize int64 `json:"actual_size"`
|
||||
ETag string `gorm:"size:255" json:"etag"`
|
||||
Status FileStatus `gorm:"size:16;not null;index:idx_files_object_status_expires_at,priority:1;index:idx_files_object_status_delete_lease,priority:1" json:"status"`
|
||||
OwnerType string `gorm:"size:64;not null;index:idx_files_object_owner_type_identity,priority:1" json:"owner_type"`
|
||||
OwnerID uint `gorm:"not null" json:"owner_id"`
|
||||
OwnerIdentity string `gorm:"size:64;not null;index:idx_files_object_owner_type_identity,priority:2" json:"owner_identity"`
|
||||
ExpiresAt *time.Time `gorm:"index:idx_files_object_status_expires_at,priority:2" json:"expires_at"`
|
||||
CompletedAt *time.Time `json:"completed_at"`
|
||||
DeleteToken string `gorm:"size:26" json:"-"`
|
||||
DeleteLeaseUntil *time.Time `gorm:"index:idx_files_object_status_delete_lease,priority:2" json:"-"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
DeletedAt gorm.DeletedAt `gorm:"index" json:"deleted_at,omitempty"`
|
||||
}
|
||||
|
||||
func (FileObject) TableName() string {
|
||||
return "files_object"
|
||||
}
|
||||
7
internal/models/query.go
Normal file
7
internal/models/query.go
Normal file
@@ -0,0 +1,7 @@
|
||||
package models
|
||||
|
||||
import "git.apinb.com/ops/files/internal/impl"
|
||||
|
||||
func InitData() error {
|
||||
return impl.DBService.AutoMigrate(&FileObject{})
|
||||
}
|
||||
27
internal/routers/register.go
Normal file
27
internal/routers/register.go
Normal file
@@ -0,0 +1,27 @@
|
||||
package routers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"git.apinb.com/bsm-sdk/core/middleware"
|
||||
"git.apinb.com/ops/files/internal/auth"
|
||||
"git.apinb.com/ops/files/internal/logic/files"
|
||||
"git.apinb.com/ops/files/internal/logic/ping"
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
// Register 注册基础服务路由。
|
||||
func Register(srvKey string, engine *gin.Engine) {
|
||||
v1Group := engine.Group(fmt.Sprintf("/%s/v1", srvKey))
|
||||
v1Group.GET("/ping/hello", ping.Hello)
|
||||
|
||||
registerFileRoutes(v1Group.Group("", middleware.JwtAuth(true)))
|
||||
registerFileRoutes(v1Group.Group("/internal", auth.ServiceAuth()))
|
||||
}
|
||||
|
||||
func registerFileRoutes(group *gin.RouterGroup) {
|
||||
group.POST("/uploads/init", files.InitUpload)
|
||||
group.POST("/uploads/:identity/complete", files.CompleteUpload)
|
||||
group.GET("/files/:identity", files.Detail)
|
||||
group.DELETE("/files/:identity", files.Delete)
|
||||
}
|
||||
31
internal/storage/client.go
Normal file
31
internal/storage/client.go
Normal file
@@ -0,0 +1,31 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"git.apinb.com/ops/files/internal/config"
|
||||
"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss"
|
||||
"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss/credentials"
|
||||
)
|
||||
|
||||
type Client struct {
|
||||
client *oss.Client
|
||||
bucket string
|
||||
publicBaseURL string
|
||||
presignTTL time.Duration
|
||||
}
|
||||
|
||||
func New(cfg config.ObjectStorageConf) (*Client, error) {
|
||||
ossConfig := oss.LoadDefaultConfig().
|
||||
WithRegion(cfg.Region).
|
||||
WithEndpoint(cfg.Endpoint).
|
||||
WithCredentialsProvider(credentials.NewStaticCredentialsProvider(cfg.AccessKeyID, cfg.AccessKeySecret)).
|
||||
WithSignatureVersion(oss.SignatureVersionV4)
|
||||
|
||||
return &Client{
|
||||
client: oss.NewClient(ossConfig),
|
||||
bucket: cfg.Bucket,
|
||||
publicBaseURL: cfg.PublicBaseURL,
|
||||
presignTTL: time.Duration(cfg.PresignTTLSeconds) * time.Second,
|
||||
}, nil
|
||||
}
|
||||
76
internal/storage/object.go
Normal file
76
internal/storage/object.go
Normal file
@@ -0,0 +1,76 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss"
|
||||
)
|
||||
|
||||
type ObjectInfo struct {
|
||||
Size int64
|
||||
ETag string
|
||||
LastModified time.Time
|
||||
}
|
||||
|
||||
func (c *Client) Stat(ctx context.Context, objectKey string) (ObjectInfo, error) {
|
||||
if strings.TrimSpace(objectKey) == "" {
|
||||
return ObjectInfo{}, fmt.Errorf("对象键不能为空")
|
||||
}
|
||||
|
||||
result, err := c.client.GetObjectMeta(ctx, &oss.GetObjectMetaRequest{
|
||||
Bucket: oss.Ptr(c.bucket),
|
||||
Key: oss.Ptr(objectKey),
|
||||
})
|
||||
if err != nil {
|
||||
return ObjectInfo{}, fmt.Errorf("查询对象元数据: %w", err)
|
||||
}
|
||||
if result == nil {
|
||||
return ObjectInfo{}, fmt.Errorf("对象元数据为空")
|
||||
}
|
||||
if result.ContentLength < 0 {
|
||||
return ObjectInfo{}, fmt.Errorf("对象大小无效: %d", result.ContentLength)
|
||||
}
|
||||
if result.ETag == nil {
|
||||
return ObjectInfo{}, fmt.Errorf("对象 ETag 缺失")
|
||||
}
|
||||
etag := strings.TrimSpace(strings.Trim(strings.TrimSpace(*result.ETag), "\""))
|
||||
if etag == "" {
|
||||
return ObjectInfo{}, fmt.Errorf("对象 ETag 无效")
|
||||
}
|
||||
if result.LastModified == nil || result.LastModified.IsZero() {
|
||||
return ObjectInfo{}, fmt.Errorf("对象最后修改时间缺失")
|
||||
}
|
||||
|
||||
return ObjectInfo{
|
||||
Size: result.ContentLength,
|
||||
ETag: etag,
|
||||
LastModified: *result.LastModified,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *Client) Delete(ctx context.Context, objectKey string) error {
|
||||
if strings.TrimSpace(objectKey) == "" {
|
||||
return fmt.Errorf("对象键不能为空")
|
||||
}
|
||||
|
||||
_, err := c.client.DeleteObject(ctx, &oss.DeleteObjectRequest{
|
||||
Bucket: oss.Ptr(c.bucket),
|
||||
Key: oss.Ptr(objectKey),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("删除对象: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) PublicURL(objectKey string) string {
|
||||
return strings.TrimRight(c.publicBaseURL, "/") + "/" + strings.TrimLeft(objectKey, "/")
|
||||
}
|
||||
|
||||
func (c *Client) Bucket() string {
|
||||
return c.bucket
|
||||
}
|
||||
43
internal/storage/presign.go
Normal file
43
internal/storage/presign.go
Normal file
@@ -0,0 +1,43 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss"
|
||||
)
|
||||
|
||||
type UploadInstruction struct {
|
||||
Method string `json:"method"`
|
||||
URL string `json:"url"`
|
||||
Headers map[string]string `json:"headers"`
|
||||
ExpiresAt time.Time `json:"expires_at"`
|
||||
}
|
||||
|
||||
func (c *Client) PresignPut(ctx context.Context, objectKey, contentType string) (UploadInstruction, error) {
|
||||
if strings.TrimSpace(objectKey) == "" {
|
||||
return UploadInstruction{}, fmt.Errorf("对象键不能为空")
|
||||
}
|
||||
if strings.TrimSpace(contentType) == "" {
|
||||
return UploadInstruction{}, fmt.Errorf("内容类型不能为空")
|
||||
}
|
||||
|
||||
result, err := c.client.Presign(ctx, &oss.PutObjectRequest{
|
||||
Bucket: oss.Ptr(c.bucket),
|
||||
Key: oss.Ptr(objectKey),
|
||||
ContentType: oss.Ptr(contentType),
|
||||
ForbidOverwrite: oss.Ptr("true"),
|
||||
}, oss.PresignExpires(c.presignTTL))
|
||||
if err != nil {
|
||||
return UploadInstruction{}, fmt.Errorf("生成上传预签名: %w", err)
|
||||
}
|
||||
|
||||
return UploadInstruction{
|
||||
Method: result.Method,
|
||||
URL: result.URL,
|
||||
Headers: result.SignedHeaders,
|
||||
ExpiresAt: result.Expiration,
|
||||
}, nil
|
||||
}
|
||||
Reference in New Issue
Block a user