diff --git a/modules/setting/service.go b/modules/setting/service.go
index 03225f566..597fecee1 100644
--- a/modules/setting/service.go
+++ b/modules/setting/service.go
@@ -188,15 +188,33 @@ func loadServiceFrom(rootCfg ConfigProvider) {
 	Service.EnableUserHeatmap = sec.Key("ENABLE_USER_HEATMAP").MustBool(true)
 	Service.AutoWatchNewRepos = sec.Key("AUTO_WATCH_NEW_REPOS").MustBool(true)
 	Service.AutoWatchOnChanges = sec.Key("AUTO_WATCH_ON_CHANGES").MustBool(false)
-	Service.DefaultUserVisibility = sec.Key("DEFAULT_USER_VISIBILITY").In("public", structs.ExtractKeysFromMapString(structs.VisibilityModes))
-	Service.DefaultUserVisibilityMode = structs.VisibilityModes[Service.DefaultUserVisibility]
-	Service.AllowedUserVisibilityModes = sec.Key("ALLOWED_USER_VISIBILITY_MODES").Strings(",")
-	if len(Service.AllowedUserVisibilityModes) != 0 {
+	modes := sec.Key("ALLOWED_USER_VISIBILITY_MODES").Strings(",")
+	if len(modes) != 0 {
+		Service.AllowedUserVisibilityModes = []string{}
 		Service.AllowedUserVisibilityModesSlice = []bool{false, false, false}
-		for _, sMode := range Service.AllowedUserVisibilityModes {
-			Service.AllowedUserVisibilityModesSlice[structs.VisibilityModes[sMode]] = true
+		for _, sMode := range modes {
+			if tp, ok := structs.VisibilityModes[sMode]; ok { // remove unsupported modes
+				Service.AllowedUserVisibilityModes = append(Service.AllowedUserVisibilityModes, sMode)
+				Service.AllowedUserVisibilityModesSlice[tp] = true
+			} else {
+				log.Warn("ALLOWED_USER_VISIBILITY_MODES %s is unsupported", sMode)
+			}
 		}
 	}
+
+	if len(Service.AllowedUserVisibilityModes) == 0 {
+		Service.AllowedUserVisibilityModes = []string{"public", "limited", "private"}
+		Service.AllowedUserVisibilityModesSlice = []bool{true, true, true}
+	}
+
+	Service.DefaultUserVisibility = sec.Key("DEFAULT_USER_VISIBILITY").String()
+	if Service.DefaultUserVisibility == "" {
+		Service.DefaultUserVisibility = Service.AllowedUserVisibilityModes[0]
+	} else if !Service.AllowedUserVisibilityModesSlice[structs.VisibilityModes[Service.DefaultUserVisibility]] {
+		log.Warn("DEFAULT_USER_VISIBILITY %s is wrong or not in ALLOWED_USER_VISIBILITY_MODES, using first allowed", Service.DefaultUserVisibility)
+		Service.DefaultUserVisibility = Service.AllowedUserVisibilityModes[0]
+	}
+	Service.DefaultUserVisibilityMode = structs.VisibilityModes[Service.DefaultUserVisibility]
 	Service.DefaultOrgVisibility = sec.Key("DEFAULT_ORG_VISIBILITY").In("public", structs.ExtractKeysFromMapString(structs.VisibilityModes))
 	Service.DefaultOrgVisibilityMode = structs.VisibilityModes[Service.DefaultOrgVisibility]
 	Service.DefaultOrgMemberVisible = sec.Key("DEFAULT_ORG_MEMBER_VISIBLE").MustBool()
diff --git a/modules/setting/service_test.go b/modules/setting/service_test.go
index 656e759f4..1647bcec1 100644
--- a/modules/setting/service_test.go
+++ b/modules/setting/service_test.go
@@ -6,6 +6,8 @@ package setting
 import (
 	"testing"
 
+	"code.gitea.io/gitea/modules/structs"
+
 	"github.com/gobwas/glob"
 	"github.com/stretchr/testify/assert"
 )
@@ -44,3 +46,87 @@ EMAIL_DOMAIN_BLOCKLIST = d3, *.b
 	assert.True(t, match(Service.EmailDomainBlockList, "foo.b"))
 	assert.False(t, match(Service.EmailDomainBlockList, "d1"))
 }
+
+func TestLoadServiceVisibilityModes(t *testing.T) {
+	oldService := Service
+	defer func() {
+		Service = oldService
+	}()
+
+	kases := map[string]func(){
+		`
+[service]
+DEFAULT_USER_VISIBILITY = public
+ALLOWED_USER_VISIBILITY_MODES = public,limited,private
+`: func() {
+			assert.Equal(t, "public", Service.DefaultUserVisibility)
+			assert.Equal(t, structs.VisibleTypePublic, Service.DefaultUserVisibilityMode)
+			assert.Equal(t, []string{"public", "limited", "private"}, Service.AllowedUserVisibilityModes)
+		},
+		`
+		[service]
+		DEFAULT_USER_VISIBILITY = public
+		`: func() {
+			assert.Equal(t, "public", Service.DefaultUserVisibility)
+			assert.Equal(t, structs.VisibleTypePublic, Service.DefaultUserVisibilityMode)
+			assert.Equal(t, []string{"public", "limited", "private"}, Service.AllowedUserVisibilityModes)
+		},
+		`
+		[service]
+		DEFAULT_USER_VISIBILITY = limited
+		`: func() {
+			assert.Equal(t, "limited", Service.DefaultUserVisibility)
+			assert.Equal(t, structs.VisibleTypeLimited, Service.DefaultUserVisibilityMode)
+			assert.Equal(t, []string{"public", "limited", "private"}, Service.AllowedUserVisibilityModes)
+		},
+		`
+[service]
+ALLOWED_USER_VISIBILITY_MODES = public,limited,private
+`: func() {
+			assert.Equal(t, "public", Service.DefaultUserVisibility)
+			assert.Equal(t, structs.VisibleTypePublic, Service.DefaultUserVisibilityMode)
+			assert.Equal(t, []string{"public", "limited", "private"}, Service.AllowedUserVisibilityModes)
+		},
+		`
+[service]
+DEFAULT_USER_VISIBILITY = public
+ALLOWED_USER_VISIBILITY_MODES = limited,private
+`: func() {
+			assert.Equal(t, "limited", Service.DefaultUserVisibility)
+			assert.Equal(t, structs.VisibleTypeLimited, Service.DefaultUserVisibilityMode)
+			assert.Equal(t, []string{"limited", "private"}, Service.AllowedUserVisibilityModes)
+		},
+		`
+[service]
+DEFAULT_USER_VISIBILITY = my_type
+ALLOWED_USER_VISIBILITY_MODES = limited,private
+`: func() {
+			assert.Equal(t, "limited", Service.DefaultUserVisibility)
+			assert.Equal(t, structs.VisibleTypeLimited, Service.DefaultUserVisibilityMode)
+			assert.Equal(t, []string{"limited", "private"}, Service.AllowedUserVisibilityModes)
+		},
+		`
+[service]
+DEFAULT_USER_VISIBILITY = public
+ALLOWED_USER_VISIBILITY_MODES = public, limit, privated
+`: func() {
+			assert.Equal(t, "public", Service.DefaultUserVisibility)
+			assert.Equal(t, structs.VisibleTypePublic, Service.DefaultUserVisibilityMode)
+			assert.Equal(t, []string{"public"}, Service.AllowedUserVisibilityModes)
+		},
+	}
+
+	for kase, fun := range kases {
+		t.Run(kase, func(t *testing.T) {
+			cfg, err := NewConfigProviderFromData(kase)
+			assert.NoError(t, err)
+			loadServiceFrom(cfg)
+			fun()
+			// reset
+			Service.AllowedUserVisibilityModesSlice = []bool{true, true, true}
+			Service.AllowedUserVisibilityModes = []string{}
+			Service.DefaultUserVisibility = ""
+			Service.DefaultUserVisibilityMode = structs.VisibleTypePublic
+		})
+	}
+}