pub-solar/forgejo
5
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Allow markdown files to read from the LFS (#5787)

Browse source 
This PR makes it possible for the markdown renderer to render images and media straight from the LFS.

Fix #5746

Signed-off-by: Andrew Thornton [art27@cantab.net](mailto:art27@cantab.net)
This commit is contained in:
zeripath 2019-02-12 15:09:43 +00:00 committed by GitHub
parent 296814e887
commit 2a03e96bce
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 339 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
integrations/download_test.go
View file

@ -22,3 +22,15 @@ func TestDownloadByID(t *testing.T) {
assert.Equal(t, "# repo1\n\nDescription for repo1", resp.Body.String())
}
func TestDownloadByIDMedia(t *testing.T) {
prepareTestEnv(t)
session := loginUser(t, "user2")
// Request raw blob
req := NewRequest(t, "GET", "/user2/repo1/media/blob/4b4851ad51df6a7d9f25c979345979eaeb5b349f")
resp := session.MakeRequest(t, req, http.StatusOK)
assert.Equal(t, "# repo1\n\nDescription for repo1", resp.Body.String())
}

126
integrations/git_test.go
View file

@ -8,13 +8,16 @@ import (
"crypto/rand"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"os"
"path"
"path/filepath"
"testing"
"time"
"code.gitea.io/git"
"code.gitea.io/gitea/models"
"github.com/stretchr/testify/assert"
)
@ -39,6 +42,8 @@ func testGit(t *testing.T, u *url.URL) {
httpContext.Reponame = "repo-tmp-17"
dstPath, err := ioutil.TempDir("", httpContext.Reponame)
var little, big, littleLFS, bigLFS string
assert.NoError(t, err)
defer os.RemoveAll(dstPath)
t.Run("Standard", func(t *testing.T) {
@ -53,10 +58,10 @@ func testGit(t *testing.T, u *url.URL) {
t.Run("PushCommit", func(t *testing.T) {
t.Run("Little", func(t *testing.T) {
commitAndPush(t, littleSize, dstPath)
little = commitAndPush(t, littleSize, dstPath)
})
t.Run("Big", func(t *testing.T) {
commitAndPush(t, bigSize, dstPath)
big = commitAndPush(t, bigSize, dstPath)
})
})
})
@ -71,16 +76,60 @@ func testGit(t *testing.T, u *url.URL) {
assert.NoError(t, err)
t.Run("Little", func(t *testing.T) {
commitAndPush(t, littleSize, dstPath)
littleLFS = commitAndPush(t, littleSize, dstPath)
})
t.Run("Big", func(t *testing.T) {
commitAndPush(t, bigSize, dstPath)
bigLFS = commitAndPush(t, bigSize, dstPath)
})
})
t.Run("Locks", func(t *testing.T) {
lockTest(t, u.String(), dstPath)
})
})
t.Run("Raw", func(t *testing.T) {
session := loginUser(t, "user2")
// Request raw paths
req := NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/raw/branch/master/", little))
resp := session.MakeRequest(t, req, http.StatusOK)
assert.Equal(t, littleSize, resp.Body.Len())
req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/raw/branch/master/", big))
nilResp := session.MakeRequestNilResponseRecorder(t, req, http.StatusOK)
assert.Equal(t, bigSize, nilResp.Length)
req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/raw/branch/master/", littleLFS))
resp = session.MakeRequest(t, req, http.StatusOK)
assert.NotEqual(t, littleSize, resp.Body.Len())
assert.Contains(t, resp.Body.String(), models.LFSMetaFileIdentifier)
req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/raw/branch/master/", bigLFS))
resp = session.MakeRequest(t, req, http.StatusOK)
assert.NotEqual(t, bigSize, resp.Body.Len())
assert.Contains(t, resp.Body.String(), models.LFSMetaFileIdentifier)
})
t.Run("Media", func(t *testing.T) {
session := loginUser(t, "user2")
// Request media paths
req := NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/media/branch/master/", little))
resp := session.MakeRequestNilResponseRecorder(t, req, http.StatusOK)
assert.Equal(t, littleSize, resp.Length)
req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/media/branch/master/", big))
resp = session.MakeRequestNilResponseRecorder(t, req, http.StatusOK)
assert.Equal(t, bigSize, resp.Length)
req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/media/branch/master/", littleLFS))
resp = session.MakeRequestNilResponseRecorder(t, req, http.StatusOK)
assert.Equal(t, littleSize, resp.Length)
req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/media/branch/master/", bigLFS))
resp = session.MakeRequestNilResponseRecorder(t, req, http.StatusOK)
assert.Equal(t, bigSize, resp.Length)
})
})
t.Run("SSH", func(t *testing.T) {
sshContext := baseAPITestContext
@ -97,6 +146,7 @@ func testGit(t *testing.T, u *url.URL) {
dstPath, err := ioutil.TempDir("", sshContext.Reponame)
assert.NoError(t, err)
defer os.RemoveAll(dstPath)
var little, big, littleLFS, bigLFS string
t.Run("Standard", func(t *testing.T) {
t.Run("CreateRepo", doAPICreateRepository(sshContext, false))
@ -107,10 +157,10 @@ func testGit(t *testing.T, u *url.URL) {
//time.Sleep(5 * time.Minute)
t.Run("PushCommit", func(t *testing.T) {
t.Run("Little", func(t *testing.T) {
commitAndPush(t, littleSize, dstPath)
little = commitAndPush(t, littleSize, dstPath)
})
t.Run("Big", func(t *testing.T) {
commitAndPush(t, bigSize, dstPath)
big = commitAndPush(t, bigSize, dstPath)
})
})
})
@ -125,16 +175,59 @@ func testGit(t *testing.T, u *url.URL) {
assert.NoError(t, err)
t.Run("Little", func(t *testing.T) {
commitAndPush(t, littleSize, dstPath)
littleLFS = commitAndPush(t, littleSize, dstPath)
})
t.Run("Big", func(t *testing.T) {
commitAndPush(t, bigSize, dstPath)
bigLFS = commitAndPush(t, bigSize, dstPath)
})
})
t.Run("Locks", func(t *testing.T) {
lockTest(t, u.String(), dstPath)
})
})
t.Run("Raw", func(t *testing.T) {
session := loginUser(t, "user2")
// Request raw paths
req := NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/raw/branch/master/", little))
resp := session.MakeRequest(t, req, http.StatusOK)
assert.Equal(t, littleSize, resp.Body.Len())
req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/raw/branch/master/", big))
resp = session.MakeRequest(t, req, http.StatusOK)
assert.Equal(t, bigSize, resp.Body.Len())
req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/raw/branch/master/", littleLFS))
resp = session.MakeRequest(t, req, http.StatusOK)
assert.NotEqual(t, littleSize, resp.Body.Len())
assert.Contains(t, resp.Body.String(), models.LFSMetaFileIdentifier)
req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/raw/branch/master/", bigLFS))
resp = session.MakeRequest(t, req, http.StatusOK)
assert.NotEqual(t, bigSize, resp.Body.Len())
assert.Contains(t, resp.Body.String(), models.LFSMetaFileIdentifier)
})
t.Run("Media", func(t *testing.T) {
session := loginUser(t, "user2")
// Request media paths
req := NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/media/branch/master/", little))
resp := session.MakeRequest(t, req, http.StatusOK)
assert.Equal(t, littleSize, resp.Body.Len())
req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/media/branch/master/", big))
resp = session.MakeRequest(t, req, http.StatusOK)
assert.Equal(t, bigSize, resp.Body.Len())
req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/media/branch/master/", littleLFS))
resp = session.MakeRequest(t, req, http.StatusOK)
assert.Equal(t, littleSize, resp.Body.Len())
req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/media/branch/master/", bigLFS))
resp = session.MakeRequest(t, req, http.StatusOK)
assert.Equal(t, bigSize, resp.Body.Len())
})
})
@ -162,34 +255,35 @@ func lockTest(t *testing.T, remote, repoPath string) {
assert.NoError(t, err)
}
func commitAndPush(t *testing.T, size int, repoPath string) {
err := generateCommitWithNewData(size, repoPath, "user2@example.com", "User Two")
func commitAndPush(t *testing.T, size int, repoPath string) string {
name, err := generateCommitWithNewData(size, repoPath, "user2@example.com", "User Two")
assert.NoError(t, err)
_, err = git.NewCommand("push").RunInDir(repoPath) //Push
assert.NoError(t, err)
return name
}
func generateCommitWithNewData(size int, repoPath, email, fullName string) error {
func generateCommitWithNewData(size int, repoPath, email, fullName string) (string, error) {
//Generate random file
data := make([]byte, size)
_, err := rand.Read(data)
if err != nil {
return err
return "", err
}
tmpFile, err := ioutil.TempFile(repoPath, "data-file-")
if err != nil {
return err
return "", err
}
defer tmpFile.Close()
_, err = tmpFile.Write(data)
if err != nil {
return err
return "", err
}
//Commit
err = git.AddChanges(repoPath, false, filepath.Base(tmpFile.Name()))
if err != nil {
return err
return "", err
}
err = git.CommitChanges(repoPath, git.CommitChangesOptions{
Committer: &git.Signature{
@ -204,5 +298,5 @@ func generateCommitWithNewData(size int, repoPath, email, fullName string) error
},
Message: fmt.Sprintf("Testing commit @ %v", time.Now()),
})
return err
return filepath.Base(tmpFile.Name()), err
}

45
integrations/integration_test.go
View file

@ -35,6 +35,23 @@ import (
var mac *macaron.Macaron
type NilResponseRecorder struct {
httptest.ResponseRecorder
Length int
}
func (n *NilResponseRecorder) Write(b []byte) (int, error) {
n.Length = n.Length + len(b)
return len(b), nil
}
// NewRecorder returns an initialized ResponseRecorder.
func NewNilResponseRecorder() *NilResponseRecorder {
return &NilResponseRecorder{
ResponseRecorder: *httptest.NewRecorder(),
}
}
func TestMain(m *testing.M) {
initIntegrationTest()
mac = routes.NewMacaron()
@ -192,6 +209,22 @@ func (s *TestSession) MakeRequest(t testing.TB, req *http.Request, expectedStatu
return resp
}
func (s *TestSession) MakeRequestNilResponseRecorder(t testing.TB, req *http.Request, expectedStatus int) *NilResponseRecorder {
baseURL, err := url.Parse(setting.AppURL)
assert.NoError(t, err)
for _, c := range s.jar.Cookies(baseURL) {
req.AddCookie(c)
}
resp := MakeRequestNilResponseRecorder(t, req, expectedStatus)
ch := http.Header{}
ch.Add("Cookie", strings.Join(resp.HeaderMap["Set-Cookie"], ";"))
cr := http.Request{Header: ch}
s.jar.SetCookies(baseURL, cr.Cookies())
return resp
}
const userPassword = "password"
var loginSessionCache = make(map[string]*TestSession, 10)
@ -305,6 +338,18 @@ func MakeRequest(t testing.TB, req *http.Request, expectedStatus int) *httptest.
return recorder
}
func MakeRequestNilResponseRecorder(t testing.TB, req *http.Request, expectedStatus int) *NilResponseRecorder {
recorder := NewNilResponseRecorder()
mac.ServeHTTP(recorder, req)
if expectedStatus != NoExpectedStatus {
if !assert.EqualValues(t, expectedStatus, recorder.Code,
"Request: %s %s", req.Method, req.URL.String()) {
logUnexpectedResponse(t, &recorder.ResponseRecorder)
}
}
return recorder
}
// logUnexpectedResponse logs the contents of an unexpected response.
func logUnexpectedResponse(t testing.TB, recorder *httptest.ResponseRecorder) {
respBytes := recorder.Body.Bytes()

69
modules/lfs/pointers.go Normal file
View file

@ -0,0 +1,69 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package lfs
import (
"io"
"strconv"
"strings"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/setting"
)
// ReadPointerFile will return a partially filled LFSMetaObject if the provided reader is a pointer file
func ReadPointerFile(reader io.Reader) (*models.LFSMetaObject, *[]byte) {
if !setting.LFS.StartServer {
return nil, nil
}
buf := make([]byte, 1024)
n, _ := reader.Read(buf)
buf = buf[:n]
if isTextFile := base.IsTextFile(buf); !isTextFile {
return nil, nil
}
return IsPointerFile(&buf), &buf
}
// IsPointerFile will return a partially filled LFSMetaObject if the provided byte slice is a pointer file
func IsPointerFile(buf *[]byte) *models.LFSMetaObject {
if !setting.LFS.StartServer {
return nil
}
headString := string(*buf)
if !strings.HasPrefix(headString, models.LFSMetaFileIdentifier) {
return nil
}
splitLines := strings.Split(headString, "\n")
if len(splitLines) < 3 {
return nil
}
oid := strings.TrimPrefix(splitLines[1], models.LFSMetaFileOidPrefix)
size, err := strconv.ParseInt(strings.TrimPrefix(splitLines[2], "size "), 10, 64)
if len(oid) != 64 || err != nil {
return nil
}
contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
meta := &models.LFSMetaObject{Oid: oid, Size: size}
if !contentStore.Exists(meta) {
return nil
}
return meta
}
// ReadMetaObject will read a models.LFSMetaObject and return a reader
func ReadMetaObject(meta *models.LFSMetaObject) (io.ReadCloser, error) {
contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
return contentStore.Get(meta, 0)
}

2
modules/markup/markdown/markdown.go
View file

@ -106,7 +106,7 @@ func (r *Renderer) Image(out *bytes.Buffer, link []byte, title []byte, alt []byt
if r.IsWiki {
prefix = util.URLJoin(prefix, "wiki", "raw")
}
prefix = strings.Replace(prefix, "/src/", "/raw/", 1)
prefix = strings.Replace(prefix, "/src/", "/media/", 1)
if len(link) > 0 && !markup.IsLink(link) {
lnk := string(link)
lnk = util.URLJoin(prefix, lnk)

1
options/locale/locale_en-US.ini
View file

@ -612,6 +612,7 @@ editor.new_file = New File
editor.upload_file = Upload File
editor.edit_file = Edit File
editor.preview_changes = Preview Changes
editor.cannot_edit_lfs_files = LFS files cannot be edited in the web interface.
editor.cannot_edit_non_text_files = Binary files cannot be edited in the web interface.
editor.edit_this_file = Edit File
editor.must_be_on_a_branch = You must be on a branch to make or propose changes to this file.

56
routers/repo/download.go
View file

@ -15,6 +15,7 @@ import (
"code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/lfs"
)
// ServeData download file from io.Reader
@ -55,6 +56,29 @@ func ServeBlob(ctx *context.Context, blob *git.Blob) error {
return ServeData(ctx, ctx.Repo.TreePath, dataRc)
}
// ServeBlobOrLFS download a git.Blob redirecting to LFS if necessary
func ServeBlobOrLFS(ctx *context.Context, blob *git.Blob) error {
dataRc, err := blob.DataAsync()
if err != nil {
return err
}
defer dataRc.Close()
if meta, _ := lfs.ReadPointerFile(dataRc); meta != nil {
meta, _ = ctx.Repo.Repository.GetLFSMetaObjectByOid(meta.Oid)
if meta == nil {
return ServeBlob(ctx, blob)
}
lfsDataRc, err := lfs.ReadMetaObject(meta)
if err != nil {
return err
}
return ServeData(ctx, ctx.Repo.TreePath, lfsDataRc)
}
return ServeBlob(ctx, blob)
}
// SingleDownload download a file by repos path
func SingleDownload(ctx *context.Context) {
blob, err := ctx.Repo.Commit.GetBlobByPath(ctx.Repo.TreePath)
@ -71,6 +95,22 @@ func SingleDownload(ctx *context.Context) {
}
}
// SingleDownloadOrLFS download a file by repos path redirecting to LFS if necessary
func SingleDownloadOrLFS(ctx *context.Context) {
blob, err := ctx.Repo.Commit.GetBlobByPath(ctx.Repo.TreePath)
if err != nil {
if git.IsErrNotExist(err) {
ctx.NotFound("GetBlobByPath", nil)
} else {
ctx.ServerError("GetBlobByPath", err)
}
return
}
if err = ServeBlobOrLFS(ctx, blob); err != nil {
ctx.ServerError("ServeBlobOrLFS", err)
}
}
// DownloadByID download a file by sha1 ID
func DownloadByID(ctx *context.Context) {
blob, err := ctx.Repo.GitRepo.GetBlob(ctx.Params("sha"))
@ -86,3 +126,19 @@ func DownloadByID(ctx *context.Context) {
ctx.ServerError("ServeBlob", err)
}
}
// DownloadByIDOrLFS download a file by sha1 ID taking account of LFS
func DownloadByIDOrLFS(ctx *context.Context) {
blob, err := ctx.Repo.GitRepo.GetBlob(ctx.Params("sha"))
if err != nil {
if git.IsErrNotExist(err) {
ctx.NotFound("GetBlob", nil)
} else {
ctx.ServerError("GetBlob", err)
}
return
}
if err = ServeBlobOrLFS(ctx, blob); err != nil {
ctx.ServerError("ServeBlob", err)
}
}

48
routers/repo/view.go
View file

@ -12,7 +12,6 @@ import (
gotemplate "html/template"
"io/ioutil"
"path"
"strconv"
"strings"
"code.gitea.io/git"
@ -179,34 +178,42 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
buf = buf[:n]
isTextFile := base.IsTextFile(buf)
isLFSFile := false
ctx.Data["IsTextFile"] = isTextFile
//Check for LFS meta file
if isTextFile && setting.LFS.StartServer {
headString := string(buf)
if strings.HasPrefix(headString, models.LFSMetaFileIdentifier) {
splitLines := strings.Split(headString, "\n")
if len(splitLines) >= 3 {
oid := strings.TrimPrefix(splitLines[1], models.LFSMetaFileOidPrefix)
size, err := strconv.ParseInt(strings.TrimPrefix(splitLines[2], "size "), 10, 64)
if len(oid) == 64 && err == nil {
contentStore := &lfs.ContentStore{BasePath: setting.LFS.ContentPath}
meta := &models.LFSMetaObject{Oid: oid}
if contentStore.Exists(meta) {
ctx.Data["IsTextFile"] = false
isTextFile = false
if isTextFile {
if meta := lfs.IsPointerFile(&buf); meta != nil {
if meta, _ = ctx.Repo.Repository.GetLFSMetaObjectByOid(meta.Oid); meta != nil {
ctx.Data["IsLFSFile"] = true
ctx.Data["FileSize"] = size
isLFSFile = true
// OK read the lfs object
dataRc, err := lfs.ReadMetaObject(meta)
if err != nil {
ctx.ServerError("ReadMetaObject", err)
return
}
defer dataRc.Close()
buf = make([]byte, 1024)
n, _ = dataRc.Read(buf)
buf = buf[:n]
isTextFile = base.IsTextFile(buf)
ctx.Data["IsTextFile"] = isTextFile
ctx.Data["FileSize"] = meta.Size
filenameBase64 := base64.RawURLEncoding.EncodeToString([]byte(blob.Name()))
ctx.Data["RawFileLink"] = fmt.Sprintf("%s%s.git/info/lfs/objects/%s/%s", setting.AppURL, ctx.Repo.Repository.FullName(), oid, filenameBase64)
}
}
ctx.Data["RawFileLink"] = fmt.Sprintf("%s%s.git/info/lfs/objects/%s/%s", setting.AppURL, ctx.Repo.Repository.FullName(), meta.Oid, filenameBase64)
}
}
}
// Assume file is not editable first.
if !isTextFile {
if isLFSFile {
ctx.Data["EditFileTooltip"] = ctx.Tr("repo.editor.cannot_edit_lfs_files")
} else if !isTextFile {
ctx.Data["EditFileTooltip"] = ctx.Tr("repo.editor.cannot_edit_non_text_files")
}
@ -263,7 +270,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
}
ctx.Data["LineNums"] = gotemplate.HTML(output.String())
}
if !isLFSFile {
if ctx.Repo.CanEnableEditor() {
ctx.Data["CanEditFile"] = true
ctx.Data["EditFileTooltip"] = ctx.Tr("repo.editor.edit_this_file")
@ -272,6 +279,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
} else if !ctx.Repo.CanWrite(models.UnitTypeCode) {
ctx.Data["EditFileTooltip"] = ctx.Tr("repo.editor.fork_before_edit")
}
}
case base.IsPDFFile(buf):
ctx.Data["IsPDFFile"] = true

9
routers/routes/routes.go
View file

@ -731,6 +731,15 @@ func RegisterRoutes(m *macaron.Macaron) {
})
}, repo.MustAllowPulls)
m.Group("/media", func() {
m.Get("/branch/*", context.RepoRefByType(context.RepoRefBranch), repo.SingleDownloadOrLFS)
m.Get("/tag/*", context.RepoRefByType(context.RepoRefTag), repo.SingleDownloadOrLFS)
m.Get("/commit/*", context.RepoRefByType(context.RepoRefCommit), repo.SingleDownloadOrLFS)
m.Get("/blob/:sha", context.RepoRefByType(context.RepoRefBlob), repo.DownloadByIDOrLFS)
// "/*" route is deprecated, and kept for backward compatibility
m.Get("/*", context.RepoRefByType(context.RepoRefLegacy), repo.SingleDownloadOrLFS)
}, repo.MustBeNotEmpty, reqRepoCodeReader)
m.Group("/raw", func() {
m.Get("/branch/*", context.RepoRefByType(context.RepoRefBranch), repo.SingleDownload)
m.Get("/tag/*", context.RepoRefByType(context.RepoRefTag), repo.SingleDownload)