pub-solar/forgejo
5
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Add package registry quota limits (#21584)

Browse source 
Related #20471

This PR adds global quota limits for the package registry. Settings for
individual users/orgs can be added in a seperate PR using the settings
table.

Co-authored-by: Lauris BH <lauris@nix.lv>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
This commit is contained in:
KN4CK3R 2022-11-09 07:34:27 +01:00 committed by GitHub
parent cb83288530
commit 20674dd05d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
20 changed files with 378 additions and 61 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
cmd/migrate_storage_test.go
View file

@ -44,8 +44,9 @@ func TestMigratePackages(t *testing.T) {
PackageFileInfo: packages_service.PackageFileInfo{
Filename: "a.go",
},
Data: buf,
IsLead: true,
Creator: creator,
Data: buf,
IsLead: true,
})
assert.NoError(t, err)
assert.NotNil(t, v)

29
custom/conf/app.example.ini
View file

@ -2335,6 +2335,35 @@ ROUTER = console
;;
;; Path for chunked uploads. Defaults to APP_DATA_PATH + `tmp/package-upload`
;CHUNKED_UPLOAD_PATH = tmp/package-upload
;;
;; Maxmimum count of package versions a single owner can have (`-1` means no limits)
;LIMIT_TOTAL_OWNER_COUNT = -1
;; Maxmimum size of packages a single owner can use (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_TOTAL_OWNER_SIZE = -1
;; Maxmimum size of a Composer upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_SIZE_COMPOSER = -1
;; Maxmimum size of a Conan upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_SIZE_CONAN = -1
;; Maxmimum size of a Container upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_SIZE_CONTAINER = -1
;; Maxmimum size of a Generic upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_SIZE_GENERIC = -1
;; Maxmimum size of a Helm upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_SIZE_HELM = -1
;; Maxmimum size of a Maven upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_SIZE_MAVEN = -1
;; Maxmimum size of a npm upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_SIZE_NPM = -1
;; Maxmimum size of a NuGet upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_SIZE_NUGET = -1
;; Maxmimum size of a Pub upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_SIZE_PUB = -1
;; Maxmimum size of a PyPI upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_SIZE_PYPI = -1
;; Maxmimum size of a RubyGems upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_SIZE_RUBYGEMS = -1
;; Maxmimum size of a Vagrant upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_SIZE_VAGRANT = -1
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

14
docs/content/doc/advanced/config-cheat-sheet.en-us.md
View file

@ -1138,6 +1138,20 @@ Task queue configuration has been moved to `queue.task`. However, the below conf
- `ENABLED`: **true**: Enable/Disable package registry capabilities
- `CHUNKED_UPLOAD_PATH`: **tmp/package-upload**: Path for chunked uploads. Defaults to `APP_DATA_PATH` + `tmp/package-upload`
- `LIMIT_TOTAL_OWNER_COUNT`: **-1**: Maxmimum count of package versions a single owner can have (`-1` means no limits)
- `LIMIT_TOTAL_OWNER_SIZE`: **-1**: Maxmimum size of packages a single owner can use (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
- `LIMIT_SIZE_COMPOSER`: **-1**: Maxmimum size of a Composer upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
- `LIMIT_SIZE_CONAN`: **-1**: Maxmimum size of a Conan upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
- `LIMIT_SIZE_CONTAINER`: **-1**: Maxmimum size of a Container upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
- `LIMIT_SIZE_GENERIC`: **-1**: Maxmimum size of a Generic upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
- `LIMIT_SIZE_HELM`: **-1**: Maxmimum size of a Helm upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
- `LIMIT_SIZE_MAVEN`: **-1**: Maxmimum size of a Maven upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
- `LIMIT_SIZE_NPM`: **-1**: Maxmimum size of a npm upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
- `LIMIT_SIZE_NUGET`: **-1**: Maxmimum size of a NuGet upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
- `LIMIT_SIZE_PUB`: **-1**: Maxmimum size of a Pub upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
- `LIMIT_SIZE_PYPI`: **-1**: Maxmimum size of a PyPI upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
- `LIMIT_SIZE_RUBYGEMS`: **-1**: Maxmimum size of a RubyGems upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
- `LIMIT_SIZE_VAGRANT`: **-1**: Maxmimum size of a Vagrant upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
## Mirror (`mirror`)

10
models/packages/package_file.go
View file

@ -199,3 +199,13 @@ func SearchFiles(ctx context.Context, opts *PackageFileSearchOptions) ([]*Packag
count, err := sess.FindAndCount(&pfs)
return pfs, count, err
}
// CalculateBlobSize sums up all blob sizes matching the search options.
// It does NOT respect the deduplication of blobs.
func CalculateBlobSize(ctx context.Context, opts *PackageFileSearchOptions) (int64, error) {
return db.GetEngine(ctx).
Table("package_file").
Where(opts.toConds()).
Join("INNER", "package_blob", "package_blob.id = package_file.blob_id").
SumInt(new(PackageBlob), "size")
}

9
models/packages/package_version.go
View file

@ -319,3 +319,12 @@ func SearchLatestVersions(ctx context.Context, opts *PackageSearchOptions) ([]*P
count, err := sess.FindAndCount(&pvs)
return pvs, count, err
}
// CountVersions counts all versions of packages matching the search options
func CountVersions(ctx context.Context, opts *PackageSearchOptions) (int64, error) {
return db.GetEngine(ctx).
Where(opts.toConds()).
Table("package_version").
Join("INNER", "package", "package.id = package_version.package_id").
Count(new(PackageVersion))
}

50
modules/setting/packages.go
View file

@ -5,11 +5,15 @@
package setting
import (
"math"
"net/url"
"os"
"path/filepath"
"code.gitea.io/gitea/modules/log"
"github.com/dustin/go-humanize"
ini "gopkg.in/ini.v1"
)
// Package registry settings
@ -19,8 +23,24 @@ var (
Enabled bool
ChunkedUploadPath string
RegistryHost string
LimitTotalOwnerCount int64
LimitTotalOwnerSize int64
LimitSizeComposer int64
LimitSizeConan int64
LimitSizeContainer int64
LimitSizeGeneric int64
LimitSizeHelm int64
LimitSizeMaven int64
LimitSizeNpm int64
LimitSizeNuGet int64
LimitSizePub int64
LimitSizePyPI int64
LimitSizeRubyGems int64
LimitSizeVagrant int64
}{
Enabled: true,
Enabled: true,
LimitTotalOwnerCount: -1,
}
)
@ -43,4 +63,32 @@ func newPackages() {
if err := os.MkdirAll(Packages.ChunkedUploadPath, os.ModePerm); err != nil {
log.Error("Unable to create chunked upload directory: %s (%v)", Packages.ChunkedUploadPath, err)
}
Packages.LimitTotalOwnerSize = mustBytes(sec, "LIMIT_TOTAL_OWNER_SIZE")
Packages.LimitSizeComposer = mustBytes(sec, "LIMIT_SIZE_COMPOSER")
Packages.LimitSizeConan = mustBytes(sec, "LIMIT_SIZE_CONAN")
Packages.LimitSizeContainer = mustBytes(sec, "LIMIT_SIZE_CONTAINER")
Packages.LimitSizeGeneric = mustBytes(sec, "LIMIT_SIZE_GENERIC")
Packages.LimitSizeHelm = mustBytes(sec, "LIMIT_SIZE_HELM")
Packages.LimitSizeMaven = mustBytes(sec, "LIMIT_SIZE_MAVEN")
Packages.LimitSizeNpm = mustBytes(sec, "LIMIT_SIZE_NPM")
Packages.LimitSizeNuGet = mustBytes(sec, "LIMIT_SIZE_NUGET")
Packages.LimitSizePub = mustBytes(sec, "LIMIT_SIZE_PUB")
Packages.LimitSizePyPI = mustBytes(sec, "LIMIT_SIZE_PYPI")
Packages.LimitSizeRubyGems = mustBytes(sec, "LIMIT_SIZE_RUBYGEMS")
Packages.LimitSizeVagrant = mustBytes(sec, "LIMIT_SIZE_VAGRANT")
}
func mustBytes(section *ini.Section, key string) int64 {
const noLimit = "-1"
value := section.Key(key).MustString(noLimit)
if value == noLimit {
return -1
}
bytes, err := humanize.ParseBytes(value)
if err != nil || bytes > math.MaxInt64 {
return -1
}
return int64(bytes)
}

31
modules/setting/packages_test.go Normal file
View file

@ -0,0 +1,31 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package setting
import (
"testing"
"github.com/stretchr/testify/assert"
ini "gopkg.in/ini.v1"
)
func TestMustBytes(t *testing.T) {
test := func(value string) int64 {
sec, _ := ini.Empty().NewSection("test")
sec.NewKey("VALUE", value)
return mustBytes(sec, "VALUE")
}
assert.EqualValues(t, -1, test(""))
assert.EqualValues(t, -1, test("-1"))
assert.EqualValues(t, 0, test("0"))
assert.EqualValues(t, 1, test("1"))
assert.EqualValues(t, 10000, test("10000"))
assert.EqualValues(t, 1000000, test("1 mb"))
assert.EqualValues(t, 1048576, test("1mib"))
assert.EqualValues(t, 1782579, test("1.7mib"))
assert.EqualValues(t, -1, test("1 yib")) // too large
}

14
routers/api/packages/composer/composer.go
View file

@ -235,16 +235,20 @@ func UploadPackage(ctx *context.Context) {
PackageFileInfo: packages_service.PackageFileInfo{
Filename: strings.ToLower(fmt.Sprintf("%s.%s.zip", strings.ReplaceAll(cp.Name, "/", "-"), cp.Version)),
},
Data: buf,
IsLead: true,
Creator: ctx.Doer,
Data: buf,
IsLead: true,
},
)
if err != nil {
if err == packages_model.ErrDuplicatePackageVersion {
switch err {
case packages_model.ErrDuplicatePackageVersion:
apiError(ctx, http.StatusBadRequest, err)
return
case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
apiError(ctx, http.StatusForbidden, err)
default:
apiError(ctx, http.StatusInternalServerError, err)
}
apiError(ctx, http.StatusInternalServerError, err)
return
}

14
routers/api/packages/conan/conan.go
View file

@ -348,8 +348,9 @@ func uploadFile(ctx *context.Context, fileFilter container.Set[string], fileKey
Filename: strings.ToLower(filename),
CompositeKey: fileKey,
},
Data: buf,
IsLead: isConanfileFile,
Creator: ctx.Doer,
Data: buf,
IsLead: isConanfileFile,
Properties: map[string]string{
conan_module.PropertyRecipeUser: rref.User,
conan_module.PropertyRecipeChannel: rref.Channel,
@ -416,11 +417,14 @@ func uploadFile(ctx *context.Context, fileFilter container.Set[string], fileKey
pfci,
)
if err != nil {
if err == packages_model.ErrDuplicatePackageFile {
switch err {
case packages_model.ErrDuplicatePackageFile:
apiError(ctx, http.StatusBadRequest, err)
return
case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
apiError(ctx, http.StatusForbidden, err)
default:
apiError(ctx, http.StatusInternalServerError, err)
}
apiError(ctx, http.StatusInternalServerError, err)
return
}

14
routers/api/packages/generic/generic.go
View file

@ -104,16 +104,20 @@ func UploadPackage(ctx *context.Context) {
PackageFileInfo: packages_service.PackageFileInfo{
Filename: filename,
},
Data: buf,
IsLead: true,
Creator: ctx.Doer,
Data: buf,
IsLead: true,
},
)
if err != nil {
if err == packages_model.ErrDuplicatePackageFile {
switch err {
case packages_model.ErrDuplicatePackageFile:
apiError(ctx, http.StatusConflict, err)
return
case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
apiError(ctx, http.StatusForbidden, err)
default:
apiError(ctx, http.StatusInternalServerError, err)
}
apiError(ctx, http.StatusInternalServerError, err)
return
}

10
routers/api/packages/helm/helm.go
View file

@ -186,17 +186,21 @@ func UploadPackage(ctx *context.Context) {
PackageFileInfo: packages_service.PackageFileInfo{
Filename: createFilename(metadata),
},
Creator: ctx.Doer,
Data: buf,
IsLead: true,
OverwriteExisting: true,
},
)
if err != nil {
if err == packages_model.ErrDuplicatePackageVersion {
switch err {
case packages_model.ErrDuplicatePackageVersion:
apiError(ctx, http.StatusConflict, err)
return
case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
apiError(ctx, http.StatusForbidden, err)
default:
apiError(ctx, http.StatusInternalServerError, err)
}
apiError(ctx, http.StatusInternalServerError, err)
return
}

10
routers/api/packages/maven/maven.go
View file

@ -266,6 +266,7 @@ func UploadPackageFile(ctx *context.Context) {
PackageFileInfo: packages_service.PackageFileInfo{
Filename: params.Filename,
},
Creator: ctx.Doer,
Data: buf,
IsLead: false,
OverwriteExisting: params.IsMeta,
@ -312,11 +313,14 @@ func UploadPackageFile(ctx *context.Context) {
pfci,
)
if err != nil {
if err == packages_model.ErrDuplicatePackageFile {
switch err {
case packages_model.ErrDuplicatePackageFile:
apiError(ctx, http.StatusBadRequest, err)
return
case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
apiError(ctx, http.StatusForbidden, err)
default:
apiError(ctx, http.StatusInternalServerError, err)
}
apiError(ctx, http.StatusInternalServerError, err)
return
}

14
routers/api/packages/npm/npm.go
View file

@ -180,16 +180,20 @@ func UploadPackage(ctx *context.Context) {
PackageFileInfo: packages_service.PackageFileInfo{
Filename: npmPackage.Filename,
},
Data: buf,
IsLead: true,
Creator: ctx.Doer,
Data: buf,
IsLead: true,
},
)
if err != nil {
if err == packages_model.ErrDuplicatePackageVersion {
switch err {
case packages_model.ErrDuplicatePackageVersion:
apiError(ctx, http.StatusBadRequest, err)
return
case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
apiError(ctx, http.StatusForbidden, err)
default:
apiError(ctx, http.StatusInternalServerError, err)
}
apiError(ctx, http.StatusInternalServerError, err)
return
}

28
routers/api/packages/nuget/nuget.go
View file

@ -374,16 +374,20 @@ func UploadPackage(ctx *context.Context) {
PackageFileInfo: packages_service.PackageFileInfo{
Filename: strings.ToLower(fmt.Sprintf("%s.%s.nupkg", np.ID, np.Version)),
},
Data: buf,
IsLead: true,
Creator: ctx.Doer,
Data: buf,
IsLead: true,
},
)
if err != nil {
if err == packages_model.ErrDuplicatePackageVersion {
switch err {
case packages_model.ErrDuplicatePackageVersion:
apiError(ctx, http.StatusConflict, err)
return
case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
apiError(ctx, http.StatusForbidden, err)
default:
apiError(ctx, http.StatusInternalServerError, err)
}
apiError(ctx, http.StatusInternalServerError, err)
return
}
@ -428,8 +432,9 @@ func UploadSymbolPackage(ctx *context.Context) {
PackageFileInfo: packages_service.PackageFileInfo{
Filename: strings.ToLower(fmt.Sprintf("%s.%s.snupkg", np.ID, np.Version)),
},
Data: buf,
IsLead: false,
Creator: ctx.Doer,
Data: buf,
IsLead: false,
},
)
if err != nil {
@ -438,6 +443,8 @@ func UploadSymbolPackage(ctx *context.Context) {
apiError(ctx, http.StatusNotFound, err)
case packages_model.ErrDuplicatePackageFile:
apiError(ctx, http.StatusConflict, err)
case packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
apiError(ctx, http.StatusForbidden, err)
default:
apiError(ctx, http.StatusInternalServerError, err)
}
@ -452,8 +459,9 @@ func UploadSymbolPackage(ctx *context.Context) {
Filename: strings.ToLower(pdb.Name),
CompositeKey: strings.ToLower(pdb.ID),
},
Data: pdb.Content,
IsLead: false,
Creator: ctx.Doer,
Data: pdb.Content,
IsLead: false,
Properties: map[string]string{
nuget_module.PropertySymbolID: strings.ToLower(pdb.ID),
},
@ -463,6 +471,8 @@ func UploadSymbolPackage(ctx *context.Context) {
switch err {
case packages_model.ErrDuplicatePackageFile:
apiError(ctx, http.StatusConflict, err)
case packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
apiError(ctx, http.StatusForbidden, err)
default:
apiError(ctx, http.StatusInternalServerError, err)
}

14
routers/api/packages/pub/pub.go
View file

@ -199,16 +199,20 @@ func UploadPackageFile(ctx *context.Context) {
PackageFileInfo: packages_service.PackageFileInfo{
Filename: strings.ToLower(pck.Version + ".tar.gz"),
},
Data: buf,
IsLead: true,
Creator: ctx.Doer,
Data: buf,
IsLead: true,
},
)
if err != nil {
if err == packages_model.ErrDuplicatePackageVersion {
switch err {
case packages_model.ErrDuplicatePackageVersion:
apiError(ctx, http.StatusBadRequest, err)
return
case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
apiError(ctx, http.StatusForbidden, err)
default:
apiError(ctx, http.StatusInternalServerError, err)
}
apiError(ctx, http.StatusInternalServerError, err)
return
}

14
routers/api/packages/pypi/pypi.go
View file

@ -162,16 +162,20 @@ func UploadPackageFile(ctx *context.Context) {
PackageFileInfo: packages_service.PackageFileInfo{
Filename: fileHeader.Filename,
},
Data: buf,
IsLead: true,
Creator: ctx.Doer,
Data: buf,
IsLead: true,
},
)
if err != nil {
if err == packages_model.ErrDuplicatePackageFile {
switch err {
case packages_model.ErrDuplicatePackageFile:
apiError(ctx, http.StatusBadRequest, err)
return
case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
apiError(ctx, http.StatusForbidden, err)
default:
apiError(ctx, http.StatusInternalServerError, err)
}
apiError(ctx, http.StatusInternalServerError, err)
return
}

14
routers/api/packages/rubygems/rubygems.go
View file

@ -242,16 +242,20 @@ func UploadPackageFile(ctx *context.Context) {
PackageFileInfo: packages_service.PackageFileInfo{
Filename: filename,
},
Data: buf,
IsLead: true,
Creator: ctx.Doer,
Data: buf,
IsLead: true,
},
)
if err != nil {
if err == packages_model.ErrDuplicatePackageVersion {
switch err {
case packages_model.ErrDuplicatePackageVersion:
apiError(ctx, http.StatusBadRequest, err)
return
case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
apiError(ctx, http.StatusForbidden, err)
default:
apiError(ctx, http.StatusInternalServerError, err)
}
apiError(ctx, http.StatusInternalServerError, err)
return
}

14
routers/api/packages/vagrant/vagrant.go
View file

@ -193,19 +193,23 @@ func UploadPackageFile(ctx *context.Context) {
PackageFileInfo: packages_service.PackageFileInfo{
Filename: strings.ToLower(boxProvider),
},
Data: buf,
IsLead: true,
Creator: ctx.Doer,
Data: buf,
IsLead: true,
Properties: map[string]string{
vagrant_module.PropertyProvider: strings.TrimSuffix(boxProvider, ".box"),
},
},
)
if err != nil {
if err == packages_model.ErrDuplicatePackageFile {
switch err {
case packages_model.ErrDuplicatePackageFile:
apiError(ctx, http.StatusConflict, err)
return
case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
apiError(ctx, http.StatusForbidden, err)
default:
apiError(ctx, http.StatusInternalServerError, err)
}
apiError(ctx, http.StatusInternalServerError, err)
return
}

97
services/packages/packages.go
View file

@ -6,6 +6,7 @@ package packages
import (
"context"
"errors"
"fmt"
"io"
"strings"
@ -19,10 +20,17 @@ import (
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/notification"
packages_module "code.gitea.io/gitea/modules/packages"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
container_service "code.gitea.io/gitea/services/packages/container"
)
var (
ErrQuotaTypeSize = errors.New("maximum allowed package type size exceeded")
ErrQuotaTotalSize = errors.New("maximum allowed package storage quota exceeded")
ErrQuotaTotalCount = errors.New("maximum allowed package count exceeded")
)
// PackageInfo describes a package
type PackageInfo struct {
Owner *user_model.User
@ -50,6 +58,7 @@ type PackageFileInfo struct {
// PackageFileCreationInfo describes a package file to create
type PackageFileCreationInfo struct {
PackageFileInfo
Creator *user_model.User
Data packages_module.HashedSizeReader
IsLead bool
Properties map[string]string
@ -78,7 +87,7 @@ func createPackageAndAddFile(pvci *PackageCreationInfo, pfci *PackageFileCreatio
return nil, nil, err
}
pf, pb, blobCreated, err := addFileToPackageVersion(ctx, pv, pfci)
pf, pb, blobCreated, err := addFileToPackageVersion(ctx, pv, &pvci.PackageInfo, pfci)
removeBlob := false
defer func() {
if blobCreated && removeBlob {
@ -164,6 +173,10 @@ func createPackageAndVersion(ctx context.Context, pvci *PackageCreationInfo, all
}
if versionCreated {
if err := checkCountQuotaExceeded(ctx, pvci.Creator, pvci.Owner); err != nil {
return nil, false, err
}
for name, value := range pvci.VersionProperties {
if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeVersion, pv.ID, name, value); err != nil {
log.Error("Error setting package version property: %v", err)
@ -188,7 +201,7 @@ func AddFileToExistingPackage(pvi *PackageInfo, pfci *PackageFileCreationInfo) (
return nil, nil, err
}
pf, pb, blobCreated, err := addFileToPackageVersion(ctx, pv, pfci)
pf, pb, blobCreated, err := addFileToPackageVersion(ctx, pv, pvi, pfci)
removeBlob := false
defer func() {
if removeBlob {
@ -224,9 +237,13 @@ func NewPackageBlob(hsr packages_module.HashedSizeReader) *packages_model.Packag
}
}
func addFileToPackageVersion(ctx context.Context, pv *packages_model.PackageVersion, pfci *PackageFileCreationInfo) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error) {
func addFileToPackageVersion(ctx context.Context, pv *packages_model.PackageVersion, pvi *PackageInfo, pfci *PackageFileCreationInfo) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error) {
log.Trace("Adding package file: %v, %s", pv.ID, pfci.Filename)
if err := checkSizeQuotaExceeded(ctx, pfci.Creator, pvi.Owner, pvi.PackageType, pfci.Data.Size()); err != nil {
return nil, nil, false, err
}
pb, exists, err := packages_model.GetOrInsertBlob(ctx, NewPackageBlob(pfci.Data))
if err != nil {
log.Error("Error inserting package blob: %v", err)
@ -285,6 +302,80 @@ func addFileToPackageVersion(ctx context.Context, pv *packages_model.PackageVers
return pf, pb, !exists, nil
}
func checkCountQuotaExceeded(ctx context.Context, doer, owner *user_model.User) error {
if doer.IsAdmin {
return nil
}
if setting.Packages.LimitTotalOwnerCount > -1 {
totalCount, err := packages_model.CountVersions(ctx, &packages_model.PackageSearchOptions{
OwnerID: owner.ID,
IsInternal: util.OptionalBoolFalse,
})
if err != nil {
log.Error("CountVersions failed: %v", err)
return err
}
if totalCount > setting.Packages.LimitTotalOwnerCount {
return ErrQuotaTotalCount
}
}
return nil
}
func checkSizeQuotaExceeded(ctx context.Context, doer, owner *user_model.User, packageType packages_model.Type, uploadSize int64) error {
if doer.IsAdmin {
return nil
}
var typeSpecificSize int64
switch packageType {
case packages_model.TypeComposer:
typeSpecificSize = setting.Packages.LimitSizeComposer
case packages_model.TypeConan:
typeSpecificSize = setting.Packages.LimitSizeConan
case packages_model.TypeContainer:
typeSpecificSize = setting.Packages.LimitSizeContainer
case packages_model.TypeGeneric:
typeSpecificSize = setting.Packages.LimitSizeGeneric
case packages_model.TypeHelm:
typeSpecificSize = setting.Packages.LimitSizeHelm
case packages_model.TypeMaven:
typeSpecificSize = setting.Packages.LimitSizeMaven
case packages_model.TypeNpm:
typeSpecificSize = setting.Packages.LimitSizeNpm
case packages_model.TypeNuGet:
typeSpecificSize = setting.Packages.LimitSizeNuGet
case packages_model.TypePub:
typeSpecificSize = setting.Packages.LimitSizePub
case packages_model.TypePyPI:
typeSpecificSize = setting.Packages.LimitSizePyPI
case packages_model.TypeRubyGems:
typeSpecificSize = setting.Packages.LimitSizeRubyGems
case packages_model.TypeVagrant:
typeSpecificSize = setting.Packages.LimitSizeVagrant
}
if typeSpecificSize > -1 && typeSpecificSize < uploadSize {
return ErrQuotaTypeSize
}
if setting.Packages.LimitTotalOwnerSize > -1 {
totalSize, err := packages_model.CalculateBlobSize(ctx, &packages_model.PackageFileSearchOptions{
OwnerID: owner.ID,
})
if err != nil {
log.Error("CalculateBlobSize failed: %v", err)
return err
}
if totalSize+uploadSize > setting.Packages.LimitTotalOwnerSize {
return ErrQuotaTotalSize
}
}
return nil
}
// RemovePackageVersionByNameAndVersion deletes a package version and all associated files
func RemovePackageVersionByNameAndVersion(doer *user_model.User, pvi *PackageInfo) error {
pv, err := packages_model.GetVersionByNameAndVersion(db.DefaultContext, pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version)

34
tests/integration/api_packages_test.go
View file

@ -16,6 +16,7 @@ import (
container_model "code.gitea.io/gitea/models/packages/container"
"code.gitea.io/gitea/models/unittest"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/setting"
api "code.gitea.io/gitea/modules/structs"
packages_service "code.gitea.io/gitea/services/packages"
"code.gitea.io/gitea/tests"
@ -166,6 +167,39 @@ func TestPackageAccess(t *testing.T) {
uploadPackage(admin, user, http.StatusCreated)
}
func TestPackageQuota(t *testing.T) {
defer tests.PrepareTestEnv(t)()
limitTotalOwnerCount, limitTotalOwnerSize, limitSizeGeneric := setting.Packages.LimitTotalOwnerCount, setting.Packages.LimitTotalOwnerSize, setting.Packages.LimitSizeGeneric
admin := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1})
user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 10})
uploadPackage := func(doer *user_model.User, version string, expectedStatus int) {
url := fmt.Sprintf("/api/packages/%s/generic/test-package/%s/file.bin", user.Name, version)
req := NewRequestWithBody(t, "PUT", url, bytes.NewReader([]byte{1}))
AddBasicAuthHeader(req, doer.Name)
MakeRequest(t, req, expectedStatus)
}
// Exceeded quota result in StatusForbidden for normal users but admins are always allowed to upload.
setting.Packages.LimitTotalOwnerCount = 0
uploadPackage(user, "1.0", http.StatusForbidden)
uploadPackage(admin, "1.0", http.StatusCreated)
setting.Packages.LimitTotalOwnerCount = limitTotalOwnerCount
setting.Packages.LimitTotalOwnerSize = 0
uploadPackage(user, "1.1", http.StatusForbidden)
uploadPackage(admin, "1.1", http.StatusCreated)
setting.Packages.LimitTotalOwnerSize = limitTotalOwnerSize
setting.Packages.LimitSizeGeneric = 0
uploadPackage(user, "1.2", http.StatusForbidden)
uploadPackage(admin, "1.2", http.StatusCreated)
setting.Packages.LimitSizeGeneric = limitSizeGeneric
}
func TestPackageCleanup(t *testing.T) {
defer tests.PrepareTestEnv(t)()