Use native git variants by default with go-git variants as build tag (#13673)

* Move last commit cache back into modules/git

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Remove go-git from the interface for last commit cache

Signed-off-by: Andrew Thornton <art27@cantab.net>

* move cacheref to last_commit_cache

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Remove go-git from routers/private/hook

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Move FindLFSFiles to pipeline

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Make no-go-git variants

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Submodule RefID

Signed-off-by: Andrew Thornton <art27@cantab.net>

* fix issue with GetCommitsInfo

Signed-off-by: Andrew Thornton <art27@cantab.net>

* fix GetLastCommitForPaths

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Improve efficiency

Signed-off-by: Andrew Thornton <art27@cantab.net>

* More efficiency

Signed-off-by: Andrew Thornton <art27@cantab.net>

* even faster

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Reduce duplication

* As per @lunny

Signed-off-by: Andrew Thornton <art27@cantab.net>

* attempt to fix drone

Signed-off-by: Andrew Thornton <art27@cantab.net>

* fix test-tags

Signed-off-by: Andrew Thornton <art27@cantab.net>

* default to use no-go-git variants and add gogit build tag

Signed-off-by: Andrew Thornton <art27@cantab.net>

* placate lint

Signed-off-by: Andrew Thornton <art27@cantab.net>

* as per @6543

Signed-off-by: Andrew Thornton <art27@cantab.net>

Co-authored-by: 6543 <6543@obermui.de>
Co-authored-by: techknowlogick <techknowlogick@gitea.io>

.drone.yml

@@ -33,6 +33,16 @@ steps:
       GOSUMDB: sum.golang.org
       TAGS: bindata sqlite sqlite_unlock_notify
 
+  - name: lint-backend-gogit
+    pull: always
+    image: golang:1.15
+    commands:
+      - make lint-backend
+    environment:
+      GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not
+      GOSUMDB: sum.golang.org
+      TAGS: bindata gogit sqlite sqlite_unlock_notify
+
   - name: checks-frontend
     image: node:14
     commands:
@@ -69,7 +79,7 @@ steps:
       GOPROXY: off
       GOOS: linux
       GOARCH: arm64
-      TAGS: bindata
+      TAGS: bindata gogit
     commands:
       - make backend # test cross compile
       - rm ./gitea # clean
@@ -173,6 +183,17 @@ steps:
       GITHUB_READ_TOKEN:
         from_secret: github_read_token
 
+  - name: unit-test-gogit
+    pull: always
+    image: golang:1.15
+    commands:
+      - make unit-test-coverage test-check
+    environment:
+      GOPROXY: off
+      TAGS: bindata gogit sqlite sqlite_unlock_notify
+      GITHUB_READ_TOKEN:
+        from_secret: github_read_token
+
   - name: test-mysql
     image: golang:1.15
     commands:
@@ -305,7 +326,8 @@ steps:
       - timeout -s ABRT 40m make test-sqlite-migration test-sqlite
     environment:
       GOPROXY: off
-      TAGS: bindata
+      TAGS: bindata gogit sqlite sqlite_unlock_notify
+      TEST_TAGS: gogit sqlite sqlite_unlock_notify
       USE_REPO_TEST_DIR: 1
     depends_on:
       - build
@@ -318,7 +340,8 @@ steps:
       - timeout -s ABRT 40m make test-pgsql-migration test-pgsql
     environment:
       GOPROXY: off
-      TAGS: bindata
+      TAGS: bindata gogit
+      TEST_TAGS: gogit
       TEST_LDAP: 1
       USE_REPO_TEST_DIR: 1
     depends_on:

Makefile

@@ -110,7 +110,10 @@ TAGS ?=
 TAGS_SPLIT := $(subst $(COMMA), ,$(TAGS))
 TAGS_EVIDENCE := $(MAKE_EVIDENCE_DIR)/tags
 
+TEST_TAGS ?= sqlite sqlite_unlock_notify
+
 GO_DIRS := cmd integrations models modules routers build services vendor tools
+
 GO_SOURCES := $(wildcard *.go)
 GO_SOURCES += $(shell find $(GO_DIRS) -type f -name "*.go" -not -path modules/options/bindata.go -not -path modules/public/bindata.go -not -path modules/templates/bindata.go)
 
@@ -339,8 +342,8 @@ watch-backend: go-check
 
 .PHONY: test
 test:
-	@echo "Running go test..."
+	@echo "Running go test with -tags '$(TEST_TAGS)'..."
-	@$(GO) test $(GOTESTFLAGS) -mod=vendor -tags='sqlite sqlite_unlock_notify' $(GO_PACKAGES)
+	@$(GO) test $(GOTESTFLAGS) -mod=vendor -tags='$(TEST_TAGS)' $(GO_PACKAGES)
 
 .PHONY: test-check
 test-check:
@@ -356,8 +359,8 @@ test-check:
 
 .PHONY: test\#%
 test\#%:
-	@echo "Running go test..."
+	@echo "Running go test with -tags '$(TEST_TAGS)'..."
-	@$(GO) test -mod=vendor -tags='sqlite sqlite_unlock_notify' -run $(subst .,/,$*) $(GO_PACKAGES)
+	@$(GO) test -mod=vendor -tags='$(TEST_TAGS)' -run $(subst .,/,$*) $(GO_PACKAGES)
 
 .PHONY: coverage
 coverage:
@@ -365,8 +368,8 @@ coverage:
 
 .PHONY: unit-test-coverage
 unit-test-coverage:
-	@echo "Running unit-test-coverage..."
+	@echo "Running unit-test-coverage -tags '$(TEST_TAGS)'..."
-	@$(GO) test $(GOTESTFLAGS) -mod=vendor -tags='sqlite sqlite_unlock_notify' -cover -coverprofile coverage.out $(GO_PACKAGES) && echo "\n==>\033[32m Ok\033[m\n" || exit 1
+	@$(GO) test $(GOTESTFLAGS) -mod=vendor -tags='$(TEST_TAGS)' -cover -coverprofile coverage.out $(GO_PACKAGES) && echo "\n==>\033[32m Ok\033[m\n" || exit 1
 
 .PHONY: vendor
 vendor:
@@ -511,7 +514,7 @@ integrations.mssql.test: git-check $(GO_SOURCES)
 	$(GO) test $(GOTESTFLAGS) -mod=vendor -c code.gitea.io/gitea/integrations -o integrations.mssql.test
 
 integrations.sqlite.test: git-check $(GO_SOURCES)
-	$(GO) test $(GOTESTFLAGS) -mod=vendor -c code.gitea.io/gitea/integrations -o integrations.sqlite.test -tags 'sqlite sqlite_unlock_notify'
+	$(GO) test $(GOTESTFLAGS) -mod=vendor -c code.gitea.io/gitea/integrations -o integrations.sqlite.test -tags '$(TEST_TAGS)'
 
 integrations.cover.test: git-check $(GO_SOURCES)
 	$(GO) test $(GOTESTFLAGS) -mod=vendor -c code.gitea.io/gitea/integrations -coverpkg $(shell echo $(GO_PACKAGES) | tr ' ' ',') -o integrations.cover.test
@@ -534,7 +537,7 @@ migrations.mssql.test: $(GO_SOURCES)
 
 .PHONY: migrations.sqlite.test
 migrations.sqlite.test: $(GO_SOURCES)
-	$(GO) test $(GOTESTFLAGS) -c code.gitea.io/gitea/integrations/migration-test -o migrations.sqlite.test -tags 'sqlite sqlite_unlock_notify'
+	$(GO) test $(GOTESTFLAGS) -c code.gitea.io/gitea/integrations/migration-test -o migrations.sqlite.test -tags '$(TEST_TAGS)'
 
 .PHONY: check
 check: test

docs/content/doc/installation/from-source.en-us.md

@@ -101,6 +101,7 @@ Depending on requirements, the following build tags can be included.
 - `pam`: Enable support for PAM (Linux Pluggable Authentication Modules). Can
   be used to authenticate local users or extend authentication to methods
   available to PAM.
+* `gogit`: (EXPERIMENTAL) Use go-git variants of git commands.
 
 Bundling assets into the binary using the `bindata` build tag is recommended for
 production deployments. It is possible to serve the static assets directly via a reverse proxy,

modules/cache/cache.go

@@ -27,6 +27,24 @@ func newCache(cacheConfig setting.Cache) (mc.Cache, error) {
 	})
 }
 
+// Cache is the interface that operates the cache data.
+type Cache interface {
+	// Put puts value into cache with key and expire time.
+	Put(key string, val interface{}, timeout int64) error
+	// Get gets cached value by given key.
+	Get(key string) interface{}
+	// Delete deletes cached value by given key.
+	Delete(key string) error
+	// Incr increases cached int-type value by given key as a counter.
+	Incr(key string) error
+	// Decr decreases cached int-type value by given key as a counter.
+	Decr(key string) error
+	// IsExist returns true if cached value exists.
+	IsExist(key string) bool
+	// Flush deletes all cached data.
+	Flush() error
+}
+
 // NewContext start cache service
 func NewContext() error {
 	var err error
@@ -40,6 +58,11 @@ func NewContext() error {
 	return err
 }
 
+// GetCache returns the currently configured cache
+func GetCache() Cache {
+	return conn
+}
+
 // GetString returns the key value from cache with callback when no key exists in cache
 func GetString(key string, getFunc func() (string, error)) (string, error) {
 	if conn == nil || setting.CacheService.TTL == 0 {

modules/cache/last_commit.go

@@ -1,70 +0,0 @@
-// Copyright 2020 The Gitea Authors. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package cache
-
-import (
-	"crypto/sha256"
-	"fmt"
-
-	"code.gitea.io/gitea/modules/git"
-	"code.gitea.io/gitea/modules/log"
-
-	mc "gitea.com/macaron/cache"
-	"github.com/go-git/go-git/v5/plumbing/object"
-)
-
-// LastCommitCache represents a cache to store last commit
-type LastCommitCache struct {
-	repoPath    string
-	ttl         int64
-	repo        *git.Repository
-	commitCache map[string]*object.Commit
-	mc.Cache
-}
-
-// NewLastCommitCache creates a new last commit cache for repo
-func NewLastCommitCache(repoPath string, gitRepo *git.Repository, ttl int64) *LastCommitCache {
-	return &LastCommitCache{
-		repoPath:    repoPath,
-		repo:        gitRepo,
-		commitCache: make(map[string]*object.Commit),
-		ttl:         ttl,
-		Cache:       conn,
-	}
-}
-
-func (c LastCommitCache) getCacheKey(repoPath, ref, entryPath string) string {
-	hashBytes := sha256.Sum256([]byte(fmt.Sprintf("%s:%s:%s", repoPath, ref, entryPath)))
-	return fmt.Sprintf("last_commit:%x", hashBytes)
-}
-
-// Get get the last commit information by commit id and entry path
-func (c LastCommitCache) Get(ref, entryPath string) (*object.Commit, error) {
-	v := c.Cache.Get(c.getCacheKey(c.repoPath, ref, entryPath))
-	if vs, ok := v.(string); ok {
-		log.Trace("LastCommitCache hit level 1: [%s:%s:%s]", ref, entryPath, vs)
-		if commit, ok := c.commitCache[vs]; ok {
-			log.Trace("LastCommitCache hit level 2: [%s:%s:%s]", ref, entryPath, vs)
-			return commit, nil
-		}
-		id, err := c.repo.ConvertToSHA1(vs)
-		if err != nil {
-			return nil, err
-		}
-		commit, err := c.repo.GoGitRepo().CommitObject(id)
-		if err != nil {
-			return nil, err
-		}
-		c.commitCache[vs] = commit
-		return commit, nil
-	}
-	return nil, nil
-}
-
-// Put put the last commit id with commit and entry path
-func (c LastCommitCache) Put(ref, entryPath, commitID string) error {
-	log.Trace("LastCommitCache save: [%s:%s:%s]", ref, entryPath, commitID)
-	return c.Cache.Put(c.getCacheKey(c.repoPath, ref, entryPath), commitID, c.ttl)
-}

modules/convert/git_commit_test.go

@@ -13,7 +13,6 @@ import (
 	api "code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/util"
 
-	"github.com/go-git/go-git/v5/plumbing/object"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -21,7 +20,7 @@ func TestToCommitMeta(t *testing.T) {
 	assert.NoError(t, models.PrepareTestDatabase())
 	headRepo := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 1}).(*models.Repository)
 	sha1, _ := git.NewIDFromString("0000000000000000000000000000000000000000")
-	signature := &object.Signature{Name: "Test Signature", Email: "test@email.com", When: time.Unix(0, 0)}
+	signature := &git.Signature{Name: "Test Signature", Email: "test@email.com", When: time.Unix(0, 0)}
 	tag := &git.Tag{
 		Name:    "Test Tag",
 		ID:      sha1,

modules/git/batch_reader_nogogit.go

@@ -0,0 +1,243 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build !gogit
+
+package git
+
+import (
+	"bufio"
+	"bytes"
+	"math"
+	"strconv"
+)
+
+// ReadBatchLine reads the header line from cat-file --batch
+// We expect:
+// <sha> SP <type> SP <size> LF
+func ReadBatchLine(rd *bufio.Reader) (sha []byte, typ string, size int64, err error) {
+	sha, err = rd.ReadBytes(' ')
+	if err != nil {
+		return
+	}
+	sha = sha[:len(sha)-1]
+
+	typ, err = rd.ReadString(' ')
+	if err != nil {
+		return
+	}
+	typ = typ[:len(typ)-1]
+
+	var sizeStr string
+	sizeStr, err = rd.ReadString('\n')
+	if err != nil {
+		return
+	}
+
+	size, err = strconv.ParseInt(sizeStr[:len(sizeStr)-1], 10, 64)
+	return
+}
+
+// ReadTagObjectID reads a tag object ID hash from a cat-file --batch stream, throwing away the rest of the stream.
+func ReadTagObjectID(rd *bufio.Reader, size int64) (string, error) {
+	id := ""
+	var n int64
+headerLoop:
+	for {
+		line, err := rd.ReadBytes('\n')
+		if err != nil {
+			return "", err
+		}
+		n += int64(len(line))
+		idx := bytes.Index(line, []byte{' '})
+		if idx < 0 {
+			continue
+		}
+
+		if string(line[:idx]) == "object" {
+			id = string(line[idx+1 : len(line)-1])
+			break headerLoop
+		}
+	}
+
+	// Discard the rest of the tag
+	discard := size - n
+	for discard > math.MaxInt32 {
+		_, err := rd.Discard(math.MaxInt32)
+		if err != nil {
+			return id, err
+		}
+		discard -= math.MaxInt32
+	}
+	_, err := rd.Discard(int(discard))
+	return id, err
+}
+
+// ReadTreeID reads a tree ID from a cat-file --batch stream, throwing away the rest of the stream.
+func ReadTreeID(rd *bufio.Reader, size int64) (string, error) {
+	id := ""
+	var n int64
+headerLoop:
+	for {
+		line, err := rd.ReadBytes('\n')
+		if err != nil {
+			return "", err
+		}
+		n += int64(len(line))
+		idx := bytes.Index(line, []byte{' '})
+		if idx < 0 {
+			continue
+		}
+
+		if string(line[:idx]) == "tree" {
+			id = string(line[idx+1 : len(line)-1])
+			break headerLoop
+		}
+	}
+
+	// Discard the rest of the commit
+	discard := size - n
+	for discard > math.MaxInt32 {
+		_, err := rd.Discard(math.MaxInt32)
+		if err != nil {
+			return id, err
+		}
+		discard -= math.MaxInt32
+	}
+	_, err := rd.Discard(int(discard))
+	return id, err
+}
+
+// git tree files are a list:
+// <mode-in-ascii> SP <fname> NUL <20-byte SHA>
+//
+// Unfortunately this 20-byte notation is somewhat in conflict to all other git tools
+// Therefore we need some method to convert these 20-byte SHAs to a 40-byte SHA
+
+// constant hextable to help quickly convert between 20byte and 40byte hashes
+const hextable = "0123456789abcdef"
+
+// to40ByteSHA converts a 20-byte SHA in a 40-byte slice into a 40-byte sha in place
+// without allocations. This is at least 100x quicker that hex.EncodeToString
+// NB This requires that sha is a 40-byte slice
+func to40ByteSHA(sha []byte) []byte {
+	for i := 19; i >= 0; i-- {
+		v := sha[i]
+		vhi, vlo := v>>4, v&0x0f
+		shi, slo := hextable[vhi], hextable[vlo]
+		sha[i*2], sha[i*2+1] = shi, slo
+	}
+	return sha
+}
+
+// ParseTreeLineSkipMode reads an entry from a tree in a cat-file --batch stream
+// This simply skips the mode - saving a substantial amount of time and carefully avoids allocations - except where fnameBuf is too small.
+// It is recommended therefore to pass in an fnameBuf large enough to avoid almost all allocations
+//
+// Each line is composed of:
+// <mode-in-ascii-dropping-initial-zeros> SP <fname> NUL <20-byte SHA>
+//
+// We don't attempt to convert the 20-byte SHA to 40-byte SHA to save a lot of time
+func ParseTreeLineSkipMode(rd *bufio.Reader, fnameBuf, shaBuf []byte) (fname, sha []byte, n int, err error) {
+	var readBytes []byte
+	// Skip the Mode
+	readBytes, err = rd.ReadSlice(' ') // NB: DOES NOT ALLOCATE SIMPLY RETURNS SLICE WITHIN READER BUFFER
+	if err != nil {
+		return
+	}
+	n += len(readBytes)
+
+	// Deal with the fname
+	readBytes, err = rd.ReadSlice('\x00')
+	copy(fnameBuf, readBytes)
+	if len(fnameBuf) > len(readBytes) {
+		fnameBuf = fnameBuf[:len(readBytes)] // cut the buf the correct size
+	} else {
+		fnameBuf = append(fnameBuf, readBytes[len(fnameBuf):]...) // extend the buf and copy in the missing bits
+	}
+	for err == bufio.ErrBufferFull { // Then we need to read more
+		readBytes, err = rd.ReadSlice('\x00')
+		fnameBuf = append(fnameBuf, readBytes...) // there is little point attempting to avoid allocations here so just extend
+	}
+	n += len(fnameBuf)
+	if err != nil {
+		return
+	}
+	fnameBuf = fnameBuf[:len(fnameBuf)-1] // Drop the terminal NUL
+	fname = fnameBuf                      // set the returnable fname to the slice
+
+	// Now deal with the 20-byte SHA
+	idx := 0
+	for idx < 20 {
+		read := 0
+		read, err = rd.Read(shaBuf[idx:20])
+		n += read
+		if err != nil {
+			return
+		}
+		idx += read
+	}
+	sha = shaBuf
+	return
+}
+
+// ParseTreeLine reads an entry from a tree in a cat-file --batch stream
+// This carefully avoids allocations - except where fnameBuf is too small.
+// It is recommended therefore to pass in an fnameBuf large enough to avoid almost all allocations
+//
+// Each line is composed of:
+// <mode-in-ascii-dropping-initial-zeros> SP <fname> NUL <20-byte SHA>
+//
+// We don't attempt to convert the 20-byte SHA to 40-byte SHA to save a lot of time
+func ParseTreeLine(rd *bufio.Reader, modeBuf, fnameBuf, shaBuf []byte) (mode, fname, sha []byte, n int, err error) {
+	var readBytes []byte
+
+	// Read the Mode
+	readBytes, err = rd.ReadSlice(' ')
+	if err != nil {
+		return
+	}
+	n += len(readBytes)
+	copy(modeBuf, readBytes)
+	if len(modeBuf) > len(readBytes) {
+		modeBuf = modeBuf[:len(readBytes)]
+	} else {
+		modeBuf = append(modeBuf, readBytes[len(modeBuf):]...)
+
+	}
+	mode = modeBuf[:len(modeBuf)-1] // Drop the SP
+
+	// Deal with the fname
+	readBytes, err = rd.ReadSlice('\x00')
+	copy(fnameBuf, readBytes)
+	if len(fnameBuf) > len(readBytes) {
+		fnameBuf = fnameBuf[:len(readBytes)]
+	} else {
+		fnameBuf = append(fnameBuf, readBytes[len(fnameBuf):]...)
+	}
+	for err == bufio.ErrBufferFull {
+		readBytes, err = rd.ReadSlice('\x00')
+		fnameBuf = append(fnameBuf, readBytes...)
+	}
+	n += len(fnameBuf)
+	if err != nil {
+		return
+	}
+	fnameBuf = fnameBuf[:len(fnameBuf)-1]
+	fname = fnameBuf
+
+	// Deal with the 20-byte SHA
+	idx := 0
+	for idx < 20 {
+		read := 0
+		read, err = rd.Read(shaBuf[idx:20])
+		n += read
+		if err != nil {
+			return
+		}
+		idx += read
+	}
+	sha = shaBuf
+	return
+}

modules/git/blob.go

@@ -10,28 +10,9 @@ import (
 	"encoding/base64"
 	"io"
 	"io/ioutil"
-
-	"github.com/go-git/go-git/v5/plumbing"
 )
 
-// Blob represents a Git object.
+// This file contains common functions between the gogit and !gogit variants for git Blobs
-type Blob struct {
-	ID SHA1
-
-	gogitEncodedObj plumbing.EncodedObject
-	name            string
-}
-
-// DataAsync gets a ReadCloser for the contents of a blob without reading it all.
-// Calling the Close function on the result will discard all unread output.
-func (b *Blob) DataAsync() (io.ReadCloser, error) {
-	return b.gogitEncodedObj.Reader()
-}
-
-// Size returns the uncompressed size of the blob
-func (b *Blob) Size() int64 {
-	return b.gogitEncodedObj.Size()
-}
 
 // Name returns name of the tree entry this blob object was created from (or empty string)
 func (b *Blob) Name() string {

modules/git/blob_gogit.go

@@ -0,0 +1,33 @@
+// Copyright 2015 The Gogs Authors. All rights reserved.
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build gogit
+
+package git
+
+import (
+	"io"
+
+	"github.com/go-git/go-git/v5/plumbing"
+)
+
+// Blob represents a Git object.
+type Blob struct {
+	ID SHA1
+
+	gogitEncodedObj plumbing.EncodedObject
+	name            string
+}
+
+// DataAsync gets a ReadCloser for the contents of a blob without reading it all.
+// Calling the Close function on the result will discard all unread output.
+func (b *Blob) DataAsync() (io.ReadCloser, error) {
+	return b.gogitEncodedObj.Reader()
+}
+
+// Size returns the uncompressed size of the blob
+func (b *Blob) Size() int64 {
+	return b.gogitEncodedObj.Size()
+}

modules/git/blob_nogogit.go

@@ -0,0 +1,77 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build !gogit
+
+package git
+
+import (
+	"bufio"
+	"io"
+	"strconv"
+	"strings"
+)
+
+// Blob represents a Git object.
+type Blob struct {
+	ID SHA1
+
+	gotSize  bool
+	size     int64
+	repoPath string
+	name     string
+}
+
+// DataAsync gets a ReadCloser for the contents of a blob without reading it all.
+// Calling the Close function on the result will discard all unread output.
+func (b *Blob) DataAsync() (io.ReadCloser, error) {
+	stdoutReader, stdoutWriter := io.Pipe()
+	var err error
+
+	go func() {
+		stderr := &strings.Builder{}
+		err = NewCommand("cat-file", "--batch").RunInDirFullPipeline(b.repoPath, stdoutWriter, stderr, strings.NewReader(b.ID.String()+"\n"))
+		if err != nil {
+			err = ConcatenateError(err, stderr.String())
+			_ = stdoutWriter.CloseWithError(err)
+		} else {
+			_ = stdoutWriter.Close()
+		}
+	}()
+
+	bufReader := bufio.NewReader(stdoutReader)
+	_, _, size, err := ReadBatchLine(bufReader)
+	if err != nil {
+		stdoutReader.Close()
+		return nil, err
+	}
+
+	return &LimitedReaderCloser{
+		R: bufReader,
+		C: stdoutReader,
+		N: int64(size),
+	}, err
+}
+
+// Size returns the uncompressed size of the blob
+func (b *Blob) Size() int64 {
+	if b.gotSize {
+		return b.size
+	}
+
+	size, err := NewCommand("cat-file", "-s", b.ID.String()).RunInDir(b.repoPath)
+	if err != nil {
+		log("error whilst reading size for %s in %s. Error: %v", b.ID.String(), b.repoPath, err)
+		return 0
+	}
+
+	b.size, err = strconv.ParseInt(size[:len(size)-1], 10, 64)
+	if err != nil {
+		log("error whilst parsing size %s for %s in %s. Error: %v", size, b.ID.String(), b.repoPath, err)
+		return 0
+	}
+	b.gotSize = true
+
+	return b.size
+}

modules/git/cache.go

@@ -1,13 +0,0 @@
-// Copyright 2019 The Gitea Authors. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package git
-
-import "github.com/go-git/go-git/v5/plumbing/object"
-
-// LastCommitCache cache
-type LastCommitCache interface {
-	Get(ref, entryPath string) (*object.Commit, error)
-	Put(ref, entryPath, commitID string) error
-}

modules/git/command.go

@@ -189,7 +189,7 @@ func (c *Command) RunInDirTimeoutEnv(env []string, timeout time.Duration, dir st
 	stdout := new(bytes.Buffer)
 	stderr := new(bytes.Buffer)
 	if err := c.RunInDirTimeoutEnvPipeline(env, timeout, dir, stdout, stderr); err != nil {
-		return nil, concatenateError(err, stderr.String())
+		return nil, ConcatenateError(err, stderr.String())
 	}
 
 	if stdout.Len() > 0 {

modules/git/commit.go

@@ -19,8 +19,6 @@ import (
 	"net/http"
 	"strconv"
 	"strings"
-
-	"github.com/go-git/go-git/v5/plumbing/object"
 )
 
 // Commit represents a git commit.
@@ -43,61 +41,6 @@ type CommitGPGSignature struct {
 	Payload   string //TODO check if can be reconstruct from the rest of commit information to not have duplicate data
 }
 
-func convertPGPSignature(c *object.Commit) *CommitGPGSignature {
-	if c.PGPSignature == "" {
-		return nil
-	}
-
-	var w strings.Builder
-	var err error
-
-	if _, err = fmt.Fprintf(&w, "tree %s\n", c.TreeHash.String()); err != nil {
-		return nil
-	}
-
-	for _, parent := range c.ParentHashes {
-		if _, err = fmt.Fprintf(&w, "parent %s\n", parent.String()); err != nil {
-			return nil
-		}
-	}
-
-	if _, err = fmt.Fprint(&w, "author "); err != nil {
-		return nil
-	}
-
-	if err = c.Author.Encode(&w); err != nil {
-		return nil
-	}
-
-	if _, err = fmt.Fprint(&w, "\ncommitter "); err != nil {
-		return nil
-	}
-
-	if err = c.Committer.Encode(&w); err != nil {
-		return nil
-	}
-
-	if _, err = fmt.Fprintf(&w, "\n\n%s", c.Message); err != nil {
-		return nil
-	}
-
-	return &CommitGPGSignature{
-		Signature: c.PGPSignature,
-		Payload:   w.String(),
-	}
-}
-
-func convertCommit(c *object.Commit) *Commit {
-	return &Commit{
-		ID:            c.Hash,
-		CommitMessage: c.Message,
-		Committer:     &c.Committer,
-		Author:        &c.Author,
-		Signature:     convertPGPSignature(c),
-		Parents:       c.ParentHashes,
-	}
-}
-
 // Message returns the commit message. Same as retrieving CommitMessage directly.
 func (c *Commit) Message() string {
 	return c.CommitMessage
@@ -576,7 +519,7 @@ func GetCommitFileStatus(repoPath, commitID string) (*CommitFileStatus, error) {
 	err := NewCommand("show", "--name-status", "--pretty=format:''", commitID).RunInDirPipeline(repoPath, w, stderr)
 	w.Close() // Close writer to exit parsing goroutine
 	if err != nil {
-		return nil, concatenateError(err, stderr.String())
+		return nil, ConcatenateError(err, stderr.String())
 	}
 
 	<-done

modules/git/commit_convert_gogit.go

@@ -0,0 +1,70 @@
+// Copyright 2015 The Gogs Authors. All rights reserved.
+// Copyright 2018 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build gogit
+
+package git
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/go-git/go-git/v5/plumbing/object"
+)
+
+func convertPGPSignature(c *object.Commit) *CommitGPGSignature {
+	if c.PGPSignature == "" {
+		return nil
+	}
+
+	var w strings.Builder
+	var err error
+
+	if _, err = fmt.Fprintf(&w, "tree %s\n", c.TreeHash.String()); err != nil {
+		return nil
+	}
+
+	for _, parent := range c.ParentHashes {
+		if _, err = fmt.Fprintf(&w, "parent %s\n", parent.String()); err != nil {
+			return nil
+		}
+	}
+
+	if _, err = fmt.Fprint(&w, "author "); err != nil {
+		return nil
+	}
+
+	if err = c.Author.Encode(&w); err != nil {
+		return nil
+	}
+
+	if _, err = fmt.Fprint(&w, "\ncommitter "); err != nil {
+		return nil
+	}
+
+	if err = c.Committer.Encode(&w); err != nil {
+		return nil
+	}
+
+	if _, err = fmt.Fprintf(&w, "\n\n%s", c.Message); err != nil {
+		return nil
+	}
+
+	return &CommitGPGSignature{
+		Signature: c.PGPSignature,
+		Payload:   w.String(),
+	}
+}
+
+func convertCommit(c *object.Commit) *Commit {
+	return &Commit{
+		ID:            c.Hash,
+		CommitMessage: c.Message,
+		Committer:     &c.Committer,
+		Author:        &c.Author,
+		Signature:     convertPGPSignature(c),
+		Parents:       c.ParentHashes,
+	}
+}

modules/git/commit_info.go

@@ -4,286 +4,9 @@
 
 package git
 
-import (
+// CommitInfo describes the first commit with the provided entry
-	"path"
+type CommitInfo struct {
-
+	Entry         *TreeEntry
-	"github.com/emirpasic/gods/trees/binaryheap"
+	Commit        *Commit
-	"github.com/go-git/go-git/v5/plumbing"
+	SubModuleFile *SubModuleFile
-	"github.com/go-git/go-git/v5/plumbing/object"
-	cgobject "github.com/go-git/go-git/v5/plumbing/object/commitgraph"
-)
-
-// GetCommitsInfo gets information of all commits that are corresponding to these entries
-func (tes Entries) GetCommitsInfo(commit *Commit, treePath string, cache LastCommitCache) ([][]interface{}, *Commit, error) {
-	entryPaths := make([]string, len(tes)+1)
-	// Get the commit for the treePath itself
-	entryPaths[0] = ""
-	for i, entry := range tes {
-		entryPaths[i+1] = entry.Name()
-	}
-
-	commitNodeIndex, commitGraphFile := commit.repo.CommitNodeIndex()
-	if commitGraphFile != nil {
-		defer commitGraphFile.Close()
-	}
-
-	c, err := commitNodeIndex.Get(commit.ID)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	var revs map[string]*object.Commit
-	if cache != nil {
-		var unHitPaths []string
-		revs, unHitPaths, err = getLastCommitForPathsByCache(commit.ID.String(), treePath, entryPaths, cache)
-		if err != nil {
-			return nil, nil, err
-		}
-		if len(unHitPaths) > 0 {
-			revs2, err := GetLastCommitForPaths(c, treePath, unHitPaths)
-			if err != nil {
-				return nil, nil, err
-			}
-
-			for k, v := range revs2 {
-				if err := cache.Put(commit.ID.String(), path.Join(treePath, k), v.ID().String()); err != nil {
-					return nil, nil, err
-				}
-				revs[k] = v
-			}
-		}
-	} else {
-		revs, err = GetLastCommitForPaths(c, treePath, entryPaths)
-	}
-	if err != nil {
-		return nil, nil, err
-	}
-
-	commit.repo.gogitStorage.Close()
-
-	commitsInfo := make([][]interface{}, len(tes))
-	for i, entry := range tes {
-		if rev, ok := revs[entry.Name()]; ok {
-			entryCommit := convertCommit(rev)
-			if entry.IsSubModule() {
-				subModuleURL := ""
-				var fullPath string
-				if len(treePath) > 0 {
-					fullPath = treePath + "/" + entry.Name()
-				} else {
-					fullPath = entry.Name()
-				}
-				if subModule, err := commit.GetSubModule(fullPath); err != nil {
-					return nil, nil, err
-				} else if subModule != nil {
-					subModuleURL = subModule.URL
-				}
-				subModuleFile := NewSubModuleFile(entryCommit, subModuleURL, entry.ID.String())
-				commitsInfo[i] = []interface{}{entry, subModuleFile}
-			} else {
-				commitsInfo[i] = []interface{}{entry, entryCommit}
-			}
-		} else {
-			commitsInfo[i] = []interface{}{entry, nil}
-		}
-	}
-
-	// Retrieve the commit for the treePath itself (see above). We basically
-	// get it for free during the tree traversal and it's used for listing
-	// pages to display information about newest commit for a given path.
-	var treeCommit *Commit
-	if treePath == "" {
-		treeCommit = commit
-	} else if rev, ok := revs[""]; ok {
-		treeCommit = convertCommit(rev)
-		treeCommit.repo = commit.repo
-	}
-	return commitsInfo, treeCommit, nil
-}
-
-type commitAndPaths struct {
-	commit cgobject.CommitNode
-	// Paths that are still on the branch represented by commit
-	paths []string
-	// Set of hashes for the paths
-	hashes map[string]plumbing.Hash
-}
-
-func getCommitTree(c cgobject.CommitNode, treePath string) (*object.Tree, error) {
-	tree, err := c.Tree()
-	if err != nil {
-		return nil, err
-	}
-
-	// Optimize deep traversals by focusing only on the specific tree
-	if treePath != "" {
-		tree, err = tree.Tree(treePath)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	return tree, nil
-}
-
-func getFileHashes(c cgobject.CommitNode, treePath string, paths []string) (map[string]plumbing.Hash, error) {
-	tree, err := getCommitTree(c, treePath)
-	if err == object.ErrDirectoryNotFound {
-		// The whole tree didn't exist, so return empty map
-		return make(map[string]plumbing.Hash), nil
-	}
-	if err != nil {
-		return nil, err
-	}
-
-	hashes := make(map[string]plumbing.Hash)
-	for _, path := range paths {
-		if path != "" {
-			entry, err := tree.FindEntry(path)
-			if err == nil {
-				hashes[path] = entry.Hash
-			}
-		} else {
-			hashes[path] = tree.Hash
-		}
-	}
-
-	return hashes, nil
-}
-
-func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cache LastCommitCache) (map[string]*object.Commit, []string, error) {
-	var unHitEntryPaths []string
-	var results = make(map[string]*object.Commit)
-	for _, p := range paths {
-		lastCommit, err := cache.Get(commitID, path.Join(treePath, p))
-		if err != nil {
-			return nil, nil, err
-		}
-		if lastCommit != nil {
-			results[p] = lastCommit
-			continue
-		}
-
-		unHitEntryPaths = append(unHitEntryPaths, p)
-	}
-
-	return results, unHitEntryPaths, nil
-}
-
-// GetLastCommitForPaths returns last commit information
-func GetLastCommitForPaths(c cgobject.CommitNode, treePath string, paths []string) (map[string]*object.Commit, error) {
-	// We do a tree traversal with nodes sorted by commit time
-	heap := binaryheap.NewWith(func(a, b interface{}) int {
-		if a.(*commitAndPaths).commit.CommitTime().Before(b.(*commitAndPaths).commit.CommitTime()) {
-			return 1
-		}
-		return -1
-	})
-
-	resultNodes := make(map[string]cgobject.CommitNode)
-	initialHashes, err := getFileHashes(c, treePath, paths)
-	if err != nil {
-		return nil, err
-	}
-
-	// Start search from the root commit and with full set of paths
-	heap.Push(&commitAndPaths{c, paths, initialHashes})
-
-	for {
-		cIn, ok := heap.Pop()
-		if !ok {
-			break
-		}
-		current := cIn.(*commitAndPaths)
-
-		// Load the parent commits for the one we are currently examining
-		numParents := current.commit.NumParents()
-		var parents []cgobject.CommitNode
-		for i := 0; i < numParents; i++ {
-			parent, err := current.commit.ParentNode(i)
-			if err != nil {
-				break
-			}
-			parents = append(parents, parent)
-		}
-
-		// Examine the current commit and set of interesting paths
-		pathUnchanged := make([]bool, len(current.paths))
-		parentHashes := make([]map[string]plumbing.Hash, len(parents))
-		for j, parent := range parents {
-			parentHashes[j], err = getFileHashes(parent, treePath, current.paths)
-			if err != nil {
-				break
-			}
-
-			for i, path := range current.paths {
-				if parentHashes[j][path] == current.hashes[path] {
-					pathUnchanged[i] = true
-				}
-			}
-		}
-
-		var remainingPaths []string
-		for i, path := range current.paths {
-			// The results could already contain some newer change for the same path,
-			// so don't override that and bail out on the file early.
-			if resultNodes[path] == nil {
-				if pathUnchanged[i] {
-					// The path existed with the same hash in at least one parent so it could
-					// not have been changed in this commit directly.
-					remainingPaths = append(remainingPaths, path)
-				} else {
-					// There are few possible cases how can we get here:
-					// - The path didn't exist in any parent, so it must have been created by
-					//   this commit.
-					// - The path did exist in the parent commit, but the hash of the file has
-					//   changed.
-					// - We are looking at a merge commit and the hash of the file doesn't
-					//   match any of the hashes being merged. This is more common for directories,
-					//   but it can also happen if a file is changed through conflict resolution.
-					resultNodes[path] = current.commit
-				}
-			}
-		}
-
-		if len(remainingPaths) > 0 {
-			// Add the parent nodes along with remaining paths to the heap for further
-			// processing.
-			for j, parent := range parents {
-				// Combine remainingPath with paths available on the parent branch
-				// and make union of them
-				remainingPathsForParent := make([]string, 0, len(remainingPaths))
-				newRemainingPaths := make([]string, 0, len(remainingPaths))
-				for _, path := range remainingPaths {
-					if parentHashes[j][path] == current.hashes[path] {
-						remainingPathsForParent = append(remainingPathsForParent, path)
-					} else {
-						newRemainingPaths = append(newRemainingPaths, path)
-					}
-				}
-
-				if remainingPathsForParent != nil {
-					heap.Push(&commitAndPaths{parent, remainingPathsForParent, parentHashes[j]})
-				}
-
-				if len(newRemainingPaths) == 0 {
-					break
-				} else {
-					remainingPaths = newRemainingPaths
-				}
-			}
-		}
-	}
-
-	// Post-processing
-	result := make(map[string]*object.Commit)
-	for path, commitNode := range resultNodes {
-		var err error
-		result[path], err = commitNode.Commit()
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	return result, nil
 }

modules/git/commit_info_gogit.go

@@ -0,0 +1,291 @@
+// Copyright 2017 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build gogit
+
+package git
+
+import (
+	"path"
+
+	"github.com/emirpasic/gods/trees/binaryheap"
+	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/go-git/go-git/v5/plumbing/object"
+	cgobject "github.com/go-git/go-git/v5/plumbing/object/commitgraph"
+)
+
+// GetCommitsInfo gets information of all commits that are corresponding to these entries
+func (tes Entries) GetCommitsInfo(commit *Commit, treePath string, cache *LastCommitCache) ([]CommitInfo, *Commit, error) {
+	entryPaths := make([]string, len(tes)+1)
+	// Get the commit for the treePath itself
+	entryPaths[0] = ""
+	for i, entry := range tes {
+		entryPaths[i+1] = entry.Name()
+	}
+
+	commitNodeIndex, commitGraphFile := commit.repo.CommitNodeIndex()
+	if commitGraphFile != nil {
+		defer commitGraphFile.Close()
+	}
+
+	c, err := commitNodeIndex.Get(commit.ID)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	var revs map[string]*object.Commit
+	if cache != nil {
+		var unHitPaths []string
+		revs, unHitPaths, err = getLastCommitForPathsByCache(commit.ID.String(), treePath, entryPaths, cache)
+		if err != nil {
+			return nil, nil, err
+		}
+		if len(unHitPaths) > 0 {
+			revs2, err := GetLastCommitForPaths(c, treePath, unHitPaths)
+			if err != nil {
+				return nil, nil, err
+			}
+
+			for k, v := range revs2 {
+				if err := cache.Put(commit.ID.String(), path.Join(treePath, k), v.ID().String()); err != nil {
+					return nil, nil, err
+				}
+				revs[k] = v
+			}
+		}
+	} else {
+		revs, err = GetLastCommitForPaths(c, treePath, entryPaths)
+	}
+	if err != nil {
+		return nil, nil, err
+	}
+
+	commit.repo.gogitStorage.Close()
+
+	commitsInfo := make([]CommitInfo, len(tes))
+	for i, entry := range tes {
+		commitsInfo[i] = CommitInfo{
+			Entry: entry,
+		}
+		if rev, ok := revs[entry.Name()]; ok {
+			entryCommit := convertCommit(rev)
+			commitsInfo[i].Commit = entryCommit
+			if entry.IsSubModule() {
+				subModuleURL := ""
+				var fullPath string
+				if len(treePath) > 0 {
+					fullPath = treePath + "/" + entry.Name()
+				} else {
+					fullPath = entry.Name()
+				}
+				if subModule, err := commit.GetSubModule(fullPath); err != nil {
+					return nil, nil, err
+				} else if subModule != nil {
+					subModuleURL = subModule.URL
+				}
+				subModuleFile := NewSubModuleFile(entryCommit, subModuleURL, entry.ID.String())
+				commitsInfo[i].SubModuleFile = subModuleFile
+			}
+		}
+	}
+
+	// Retrieve the commit for the treePath itself (see above). We basically
+	// get it for free during the tree traversal and it's used for listing
+	// pages to display information about newest commit for a given path.
+	var treeCommit *Commit
+	if treePath == "" {
+		treeCommit = commit
+	} else if rev, ok := revs[""]; ok {
+		treeCommit = convertCommit(rev)
+		treeCommit.repo = commit.repo
+	}
+	return commitsInfo, treeCommit, nil
+}
+
+type commitAndPaths struct {
+	commit cgobject.CommitNode
+	// Paths that are still on the branch represented by commit
+	paths []string
+	// Set of hashes for the paths
+	hashes map[string]plumbing.Hash
+}
+
+func getCommitTree(c cgobject.CommitNode, treePath string) (*object.Tree, error) {
+	tree, err := c.Tree()
+	if err != nil {
+		return nil, err
+	}
+
+	// Optimize deep traversals by focusing only on the specific tree
+	if treePath != "" {
+		tree, err = tree.Tree(treePath)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return tree, nil
+}
+
+func getFileHashes(c cgobject.CommitNode, treePath string, paths []string) (map[string]plumbing.Hash, error) {
+	tree, err := getCommitTree(c, treePath)
+	if err == object.ErrDirectoryNotFound {
+		// The whole tree didn't exist, so return empty map
+		return make(map[string]plumbing.Hash), nil
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	hashes := make(map[string]plumbing.Hash)
+	for _, path := range paths {
+		if path != "" {
+			entry, err := tree.FindEntry(path)
+			if err == nil {
+				hashes[path] = entry.Hash
+			}
+		} else {
+			hashes[path] = tree.Hash
+		}
+	}
+
+	return hashes, nil
+}
+
+func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cache *LastCommitCache) (map[string]*object.Commit, []string, error) {
+	var unHitEntryPaths []string
+	var results = make(map[string]*object.Commit)
+	for _, p := range paths {
+		lastCommit, err := cache.Get(commitID, path.Join(treePath, p))
+		if err != nil {
+			return nil, nil, err
+		}
+		if lastCommit != nil {
+			results[p] = lastCommit.(*object.Commit)
+			continue
+		}
+
+		unHitEntryPaths = append(unHitEntryPaths, p)
+	}
+
+	return results, unHitEntryPaths, nil
+}
+
+// GetLastCommitForPaths returns last commit information
+func GetLastCommitForPaths(c cgobject.CommitNode, treePath string, paths []string) (map[string]*object.Commit, error) {
+	// We do a tree traversal with nodes sorted by commit time
+	heap := binaryheap.NewWith(func(a, b interface{}) int {
+		if a.(*commitAndPaths).commit.CommitTime().Before(b.(*commitAndPaths).commit.CommitTime()) {
+			return 1
+		}
+		return -1
+	})
+
+	resultNodes := make(map[string]cgobject.CommitNode)
+	initialHashes, err := getFileHashes(c, treePath, paths)
+	if err != nil {
+		return nil, err
+	}
+
+	// Start search from the root commit and with full set of paths
+	heap.Push(&commitAndPaths{c, paths, initialHashes})
+
+	for {
+		cIn, ok := heap.Pop()
+		if !ok {
+			break
+		}
+		current := cIn.(*commitAndPaths)
+
+		// Load the parent commits for the one we are currently examining
+		numParents := current.commit.NumParents()
+		var parents []cgobject.CommitNode
+		for i := 0; i < numParents; i++ {
+			parent, err := current.commit.ParentNode(i)
+			if err != nil {
+				break
+			}
+			parents = append(parents, parent)
+		}
+
+		// Examine the current commit and set of interesting paths
+		pathUnchanged := make([]bool, len(current.paths))
+		parentHashes := make([]map[string]plumbing.Hash, len(parents))
+		for j, parent := range parents {
+			parentHashes[j], err = getFileHashes(parent, treePath, current.paths)
+			if err != nil {
+				break
+			}
+
+			for i, path := range current.paths {
+				if parentHashes[j][path] == current.hashes[path] {
+					pathUnchanged[i] = true
+				}
+			}
+		}
+
+		var remainingPaths []string
+		for i, path := range current.paths {
+			// The results could already contain some newer change for the same path,
+			// so don't override that and bail out on the file early.
+			if resultNodes[path] == nil {
+				if pathUnchanged[i] {
+					// The path existed with the same hash in at least one parent so it could
+					// not have been changed in this commit directly.
+					remainingPaths = append(remainingPaths, path)
+				} else {
+					// There are few possible cases how can we get here:
+					// - The path didn't exist in any parent, so it must have been created by
+					//   this commit.
+					// - The path did exist in the parent commit, but the hash of the file has
+					//   changed.
+					// - We are looking at a merge commit and the hash of the file doesn't
+					//   match any of the hashes being merged. This is more common for directories,
+					//   but it can also happen if a file is changed through conflict resolution.
+					resultNodes[path] = current.commit
+				}
+			}
+		}
+
+		if len(remainingPaths) > 0 {
+			// Add the parent nodes along with remaining paths to the heap for further
+			// processing.
+			for j, parent := range parents {
+				// Combine remainingPath with paths available on the parent branch
+				// and make union of them
+				remainingPathsForParent := make([]string, 0, len(remainingPaths))
+				newRemainingPaths := make([]string, 0, len(remainingPaths))
+				for _, path := range remainingPaths {
+					if parentHashes[j][path] == current.hashes[path] {
+						remainingPathsForParent = append(remainingPathsForParent, path)
+					} else {
+						newRemainingPaths = append(newRemainingPaths, path)
+					}
+				}
+
+				if remainingPathsForParent != nil {
+					heap.Push(&commitAndPaths{parent, remainingPathsForParent, parentHashes[j]})
+				}
+
+				if len(newRemainingPaths) == 0 {
+					break
+				} else {
+					remainingPaths = newRemainingPaths
+				}
+			}
+		}
+	}
+
+	// Post-processing
+	result := make(map[string]*object.Commit)
+	for path, commitNode := range resultNodes {
+		var err error
+		result[path], err = commitNode.Commit()
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return result, nil
+}

modules/git/commit_info_nogogit.go

@@ -0,0 +1,370 @@
+// Copyright 2017 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build !gogit
+
+package git
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"math"
+	"path"
+	"sort"
+	"strings"
+)
+
+// GetCommitsInfo gets information of all commits that are corresponding to these entries
+func (tes Entries) GetCommitsInfo(commit *Commit, treePath string, cache *LastCommitCache) ([]CommitInfo, *Commit, error) {
+	entryPaths := make([]string, len(tes)+1)
+	// Get the commit for the treePath itself
+	entryPaths[0] = ""
+	for i, entry := range tes {
+		entryPaths[i+1] = entry.Name()
+	}
+
+	var err error
+
+	var revs map[string]*Commit
+	if cache != nil {
+		var unHitPaths []string
+		revs, unHitPaths, err = getLastCommitForPathsByCache(commit.ID.String(), treePath, entryPaths, cache)
+		if err != nil {
+			return nil, nil, err
+		}
+		if len(unHitPaths) > 0 {
+			sort.Strings(unHitPaths)
+			commits, err := GetLastCommitForPaths(commit, treePath, unHitPaths)
+			if err != nil {
+				return nil, nil, err
+			}
+
+			for i, found := range commits {
+				if err := cache.Put(commit.ID.String(), path.Join(treePath, unHitPaths[i]), found.ID.String()); err != nil {
+					return nil, nil, err
+				}
+				revs[unHitPaths[i]] = found
+			}
+		}
+	} else {
+		sort.Strings(entryPaths)
+		revs = map[string]*Commit{}
+		var foundCommits []*Commit
+		foundCommits, err = GetLastCommitForPaths(commit, treePath, entryPaths)
+		for i, found := range foundCommits {
+			revs[entryPaths[i]] = found
+		}
+	}
+	if err != nil {
+		return nil, nil, err
+	}
+
+	commitsInfo := make([]CommitInfo, len(tes))
+	for i, entry := range tes {
+		commitsInfo[i] = CommitInfo{
+			Entry: entry,
+		}
+		if entryCommit, ok := revs[entry.Name()]; ok {
+			commitsInfo[i].Commit = entryCommit
+			if entry.IsSubModule() {
+				subModuleURL := ""
+				var fullPath string
+				if len(treePath) > 0 {
+					fullPath = treePath + "/" + entry.Name()
+				} else {
+					fullPath = entry.Name()
+				}
+				if subModule, err := commit.GetSubModule(fullPath); err != nil {
+					return nil, nil, err
+				} else if subModule != nil {
+					subModuleURL = subModule.URL
+				}
+				subModuleFile := NewSubModuleFile(entryCommit, subModuleURL, entry.ID.String())
+				commitsInfo[i].SubModuleFile = subModuleFile
+			}
+		}
+	}
+
+	// Retrieve the commit for the treePath itself (see above). We basically
+	// get it for free during the tree traversal and it's used for listing
+	// pages to display information about newest commit for a given path.
+	var treeCommit *Commit
+	var ok bool
+	if treePath == "" {
+		treeCommit = commit
+	} else if treeCommit, ok = revs[""]; ok {
+		treeCommit.repo = commit.repo
+	}
+	return commitsInfo, treeCommit, nil
+}
+
+func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cache *LastCommitCache) (map[string]*Commit, []string, error) {
+	var unHitEntryPaths []string
+	var results = make(map[string]*Commit)
+	for _, p := range paths {
+		lastCommit, err := cache.Get(commitID, path.Join(treePath, p))
+		if err != nil {
+			return nil, nil, err
+		}
+		if lastCommit != nil {
+			results[p] = lastCommit.(*Commit)
+			continue
+		}
+
+		unHitEntryPaths = append(unHitEntryPaths, p)
+	}
+
+	return results, unHitEntryPaths, nil
+}
+
+// GetLastCommitForPaths returns last commit information
+func GetLastCommitForPaths(commit *Commit, treePath string, paths []string) ([]*Commit, error) {
+	// We read backwards from the commit to obtain all of the commits
+
+	// We'll do this by using rev-list to provide us with parent commits in order
+	revListReader, revListWriter := io.Pipe()
+	defer func() {
+		_ = revListWriter.Close()
+		_ = revListReader.Close()
+	}()
+
+	go func() {
+		stderr := strings.Builder{}
+		err := NewCommand("rev-list", "--format=%T", commit.ID.String()).RunInDirPipeline(commit.repo.Path, revListWriter, &stderr)
+		if err != nil {
+			_ = revListWriter.CloseWithError(ConcatenateError(err, (&stderr).String()))
+		} else {
+			_ = revListWriter.Close()
+		}
+	}()
+
+	// We feed the commits in order into cat-file --batch, followed by their trees and sub trees as necessary.
+	// so let's create a batch stdin and stdout
+	batchStdinReader, batchStdinWriter := io.Pipe()
+	batchStdoutReader, batchStdoutWriter := io.Pipe()
+	defer func() {
+		_ = batchStdinReader.Close()
+		_ = batchStdinWriter.Close()
+		_ = batchStdoutReader.Close()
+		_ = batchStdoutWriter.Close()
+	}()
+
+	go func() {
+		stderr := strings.Builder{}
+		err := NewCommand("cat-file", "--batch").RunInDirFullPipeline(commit.repo.Path, batchStdoutWriter, &stderr, batchStdinReader)
+		if err != nil {
+			_ = revListWriter.CloseWithError(ConcatenateError(err, (&stderr).String()))
+		} else {
+			_ = revListWriter.Close()
+		}
+	}()
+
+	// For simplicities sake we'll us a buffered reader
+	batchReader := bufio.NewReader(batchStdoutReader)
+
+	mapsize := 4096
+	if len(paths) > mapsize {
+		mapsize = len(paths)
+	}
+
+	path2idx := make(map[string]int, mapsize)
+	for i, path := range paths {
+		path2idx[path] = i
+	}
+
+	fnameBuf := make([]byte, 4096)
+	modeBuf := make([]byte, 40)
+
+	allShaBuf := make([]byte, (len(paths)+1)*20)
+	shaBuf := make([]byte, 20)
+	tmpTreeID := make([]byte, 40)
+
+	// commits is the returnable commits matching the paths provided
+	commits := make([]string, len(paths))
+	// ids are the blob/tree ids for the paths
+	ids := make([][]byte, len(paths))
+
+	// We'll use a scanner for the revList because it's simpler than a bufio.Reader
+	scan := bufio.NewScanner(revListReader)
+revListLoop:
+	for scan.Scan() {
+		// Get the next parent commit ID
+		commitID := scan.Text()
+		if !scan.Scan() {
+			break revListLoop
+		}
+		commitID = commitID[7:]
+		rootTreeID := scan.Text()
+
+		// push the tree to the cat-file --batch process
+		_, err := batchStdinWriter.Write([]byte(rootTreeID + "\n"))
+		if err != nil {
+			return nil, err
+		}
+
+		currentPath := ""
+
+		// OK if the target tree path is "" and the "" is in the paths just set this now
+		if treePath == "" && paths[0] == "" {
+			// If this is the first time we see this set the id appropriate for this paths to this tree and set the last commit to curCommit
+			if len(ids[0]) == 0 {
+				ids[0] = []byte(rootTreeID)
+				commits[0] = string(commitID)
+			} else if bytes.Equal(ids[0], []byte(rootTreeID)) {
+				commits[0] = string(commitID)
+			}
+		}
+
+	treeReadingLoop:
+		for {
+			_, _, size, err := ReadBatchLine(batchReader)
+			if err != nil {
+				return nil, err
+			}
+
+			// Handle trees
+
+			// n is counter for file position in the tree file
+			var n int64
+
+			// Two options: currentPath is the targetTreepath
+			if treePath == currentPath {
+				// We are in the right directory
+				// Parse each tree line in turn. (don't care about mode here.)
+				for n < size {
+					fname, sha, count, err := ParseTreeLineSkipMode(batchReader, fnameBuf, shaBuf)
+					shaBuf = sha
+					if err != nil {
+						return nil, err
+					}
+					n += int64(count)
+					idx, ok := path2idx[string(fname)]
+					if ok {
+						// Now if this is the first time round set the initial Blob(ish) SHA ID and the commit
+						if len(ids[idx]) == 0 {
+							copy(allShaBuf[20*(idx+1):20*(idx+2)], shaBuf)
+							ids[idx] = allShaBuf[20*(idx+1) : 20*(idx+2)]
+							commits[idx] = string(commitID)
+						} else if bytes.Equal(ids[idx], shaBuf) {
+							commits[idx] = string(commitID)
+						}
+					}
+					// FIXME: is there any order to the way strings are emitted from cat-file?
+					// if there is - then we could skip once we've passed all of our data
+				}
+				break treeReadingLoop
+			}
+
+			var treeID []byte
+
+			// We're in the wrong directory
+			// Find target directory in this directory
+			idx := len(currentPath)
+			if idx > 0 {
+				idx++
+			}
+			target := strings.SplitN(treePath[idx:], "/", 2)[0]
+
+			for n < size {
+				// Read each tree entry in turn
+				mode, fname, sha, count, err := ParseTreeLine(batchReader, modeBuf, fnameBuf, shaBuf)
+				if err != nil {
+					return nil, err
+				}
+				n += int64(count)
+
+				// if we have found the target directory
+				if bytes.Equal(fname, []byte(target)) && bytes.Equal(mode, []byte("40000")) {
+					copy(tmpTreeID, sha)
+					treeID = tmpTreeID
+					break
+				}
+			}
+
+			if n < size {
+				// Discard any remaining entries in the current tree
+				discard := size - n
+				for discard > math.MaxInt32 {
+					_, err := batchReader.Discard(math.MaxInt32)
+					if err != nil {
+						return nil, err
+					}
+					discard -= math.MaxInt32
+				}
+				_, err := batchReader.Discard(int(discard))
+				if err != nil {
+					return nil, err
+				}
+			}
+
+			// if we haven't found a treeID for the target directory our search is over
+			if len(treeID) == 0 {
+				break treeReadingLoop
+			}
+
+			// add the target to the current path
+			if idx > 0 {
+				currentPath += "/"
+			}
+			currentPath += target
+
+			// if we've now found the current path check its sha id and commit status
+			if treePath == currentPath && paths[0] == "" {
+				if len(ids[0]) == 0 {
+					copy(allShaBuf[0:20], treeID)
+					ids[0] = allShaBuf[0:20]
+					commits[0] = string(commitID)
+				} else if bytes.Equal(ids[0], treeID) {
+					commits[0] = string(commitID)
+				}
+			}
+			treeID = to40ByteSHA(treeID)
+			_, err = batchStdinWriter.Write(treeID)
+			if err != nil {
+				return nil, err
+			}
+			_, err = batchStdinWriter.Write([]byte("\n"))
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	commitsMap := make(map[string]*Commit, len(commits))
+	commitsMap[commit.ID.String()] = commit
+
+	commitCommits := make([]*Commit, len(commits))
+	for i, commitID := range commits {
+		c, ok := commitsMap[commitID]
+		if ok {
+			commitCommits[i] = c
+			continue
+		}
+
+		if len(commitID) == 0 {
+			continue
+		}
+
+		_, err := batchStdinWriter.Write([]byte(commitID + "\n"))
+		if err != nil {
+			return nil, err
+		}
+		_, typ, size, err := ReadBatchLine(batchReader)
+		if err != nil {
+			return nil, err
+		}
+		if typ != "commit" {
+			return nil, fmt.Errorf("unexpected type: %s for commit id: %s", typ, commitID)
+		}
+		c, err = CommitFromReader(commit.repo, MustIDFromString(string(commitID)), io.LimitReader(batchReader, int64(size)))
+		if err != nil {
+			return nil, err
+		}
+		commitCommits[i] = c
+	}
+
+	return commitCommits, scan.Err()
+}

modules/git/commit_info_test.go

@@ -58,17 +58,27 @@ func testGetCommitsInfo(t *testing.T, repo1 *Repository) {
 	for _, testCase := range testCases {
 		commit, err := repo1.GetCommit(testCase.CommitID)
 		assert.NoError(t, err)
+		assert.NotNil(t, commit)
+		assert.NotNil(t, commit.Tree)
+		assert.NotNil(t, commit.Tree.repo)
+
 		tree, err := commit.Tree.SubTree(testCase.Path)
+		assert.NotNil(t, tree, "tree is nil for testCase CommitID %s in Path %s", testCase.CommitID, testCase.Path)
+		assert.NotNil(t, tree.repo, "repo is nil for testCase CommitID %s in Path %s", testCase.CommitID, testCase.Path)
+
 		assert.NoError(t, err)
 		entries, err := tree.ListEntries()
 		assert.NoError(t, err)
 		commitsInfo, treeCommit, err := entries.GetCommitsInfo(commit, testCase.Path, nil)
-		assert.Equal(t, testCase.ExpectedTreeCommit, treeCommit.ID.String())
 		assert.NoError(t, err)
+		if err != nil {
+			t.FailNow()
+		}
+		assert.Equal(t, testCase.ExpectedTreeCommit, treeCommit.ID.String())
 		assert.Len(t, commitsInfo, len(testCase.ExpectedIDs))
 		for _, commitInfo := range commitsInfo {
-			entry := commitInfo[0].(*TreeEntry)
+			entry := commitInfo.Entry
-			commit := commitInfo[1].(*Commit)
+			commit := commitInfo.Commit
 			expectedID, ok := testCase.ExpectedIDs[entry.Name()]
 			if !assert.True(t, ok) {
 				continue

modules/git/commit_reader.go

@@ -9,13 +9,13 @@ import (
 	"bytes"
 	"io"
 	"strings"
-
-	"github.com/go-git/go-git/v5/plumbing"
 )
 
 // CommitFromReader will generate a Commit from a provided reader
-// We will need this to interpret commits from cat-file
+// We need this to interpret commits from cat-file or cat-file --batch
-func CommitFromReader(gitRepo *Repository, sha plumbing.Hash, reader io.Reader) (*Commit, error) {
+//
+// If used as part of a cat-file --batch stream you need to limit the reader to the correct size
+func CommitFromReader(gitRepo *Repository, sha SHA1, reader io.Reader) (*Commit, error) {
 	commit := &Commit{
 		ID: sha,
 	}
@@ -26,26 +26,20 @@ func CommitFromReader(gitRepo *Repository, sha plumbing.Hash, reader io.Reader)
 	message := false
 	pgpsig := false
 
-	scanner := bufio.NewScanner(reader)
+	bufReader, ok := reader.(*bufio.Reader)
-	// Split by '\n' but include the '\n'
+	if !ok {
-	scanner.Split(func(data []byte, atEOF bool) (advance int, token []byte, err error) {
+		bufReader = bufio.NewReader(reader)
-		if atEOF && len(data) == 0 {
-			return 0, nil, nil
 	}
-		if i := bytes.IndexByte(data, '\n'); i >= 0 {
-			// We have a full newline-terminated line.
-			return i + 1, data[0 : i+1], nil
-		}
-		// If we're at EOF, we have a final, non-terminated line. Return it.
-		if atEOF {
-			return len(data), data, nil
-		}
-		// Request more data.
-		return 0, nil, nil
-	})
 
-	for scanner.Scan() {
+readLoop:
-		line := scanner.Bytes()
+	for {
+		line, err := bufReader.ReadBytes('\n')
+		if err != nil {
+			if err == io.EOF {
+				break readLoop
+			}
+			return nil, err
+		}
 		if pgpsig {
 			if len(line) > 0 && line[0] == ' ' {
 				_, _ = signatureSB.Write(line[1:])
@@ -72,10 +66,10 @@ func CommitFromReader(gitRepo *Repository, sha plumbing.Hash, reader io.Reader)
 
 			switch string(split[0]) {
 			case "tree":
-				commit.Tree = *NewTree(gitRepo, plumbing.NewHash(string(data)))
+				commit.Tree = *NewTree(gitRepo, MustIDFromString(string(data)))
 				_, _ = payloadSB.Write(line)
 			case "parent":
-				commit.Parents = append(commit.Parents, plumbing.NewHash(string(data)))
+				commit.Parents = append(commit.Parents, MustIDFromString(string(data)))
 				_, _ = payloadSB.Write(line)
 			case "author":
 				commit.Author = &Signature{}
@@ -104,5 +98,5 @@ func CommitFromReader(gitRepo *Repository, sha plumbing.Hash, reader io.Reader)
 		commit.Signature = nil
 	}
 
-	return commit, scanner.Err()
+	return commit, nil
 }

modules/git/last_commit_cache.go

@@ -0,0 +1,29 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package git
+
+import (
+	"crypto/sha256"
+	"fmt"
+)
+
+// Cache represents a caching interface
+type Cache interface {
+	// Put puts value into cache with key and expire time.
+	Put(key string, val interface{}, timeout int64) error
+	// Get gets cached value by given key.
+	Get(key string) interface{}
+}
+
+func (c *LastCommitCache) getCacheKey(repoPath, ref, entryPath string) string {
+	hashBytes := sha256.Sum256([]byte(fmt.Sprintf("%s:%s:%s", repoPath, ref, entryPath)))
+	return fmt.Sprintf("last_commit:%x", hashBytes)
+}
+
+// Put put the last commit id with commit and entry path
+func (c *LastCommitCache) Put(ref, entryPath, commitID string) error {
+	log("LastCommitCache save: [%s:%s:%s]", ref, entryPath, commitID)
+	return c.cache.Put(c.getCacheKey(c.repoPath, ref, entryPath), commitID, c.ttl)
+}

modules/git/last_commit_cache_gogit.go

@@ -0,0 +1,113 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build gogit
+
+package git
+
+import (
+	"path"
+
+	"github.com/go-git/go-git/v5/plumbing/object"
+	cgobject "github.com/go-git/go-git/v5/plumbing/object/commitgraph"
+)
+
+// LastCommitCache represents a cache to store last commit
+type LastCommitCache struct {
+	repoPath    string
+	ttl         int64
+	repo        *Repository
+	commitCache map[string]*object.Commit
+	cache       Cache
+}
+
+// NewLastCommitCache creates a new last commit cache for repo
+func NewLastCommitCache(repoPath string, gitRepo *Repository, ttl int64, cache Cache) *LastCommitCache {
+	if cache == nil {
+		return nil
+	}
+	return &LastCommitCache{
+		repoPath:    repoPath,
+		repo:        gitRepo,
+		commitCache: make(map[string]*object.Commit),
+		ttl:         ttl,
+		cache:       cache,
+	}
+}
+
+// Get get the last commit information by commit id and entry path
+func (c *LastCommitCache) Get(ref, entryPath string) (interface{}, error) {
+	v := c.cache.Get(c.getCacheKey(c.repoPath, ref, entryPath))
+	if vs, ok := v.(string); ok {
+		log("LastCommitCache hit level 1: [%s:%s:%s]", ref, entryPath, vs)
+		if commit, ok := c.commitCache[vs]; ok {
+			log("LastCommitCache hit level 2: [%s:%s:%s]", ref, entryPath, vs)
+			return commit, nil
+		}
+		id, err := c.repo.ConvertToSHA1(vs)
+		if err != nil {
+			return nil, err
+		}
+		commit, err := c.repo.GoGitRepo().CommitObject(id)
+		if err != nil {
+			return nil, err
+		}
+		c.commitCache[vs] = commit
+		return commit, nil
+	}
+	return nil, nil
+}
+
+// CacheCommit will cache the commit from the gitRepository
+func (c *LastCommitCache) CacheCommit(commit *Commit) error {
+
+	commitNodeIndex, _ := commit.repo.CommitNodeIndex()
+
+	index, err := commitNodeIndex.Get(commit.ID)
+	if err != nil {
+		return err
+	}
+
+	return c.recursiveCache(index, &commit.Tree, "", 1)
+}
+
+func (c *LastCommitCache) recursiveCache(index cgobject.CommitNode, tree *Tree, treePath string, level int) error {
+	if level == 0 {
+		return nil
+	}
+
+	entries, err := tree.ListEntries()
+	if err != nil {
+		return err
+	}
+
+	entryPaths := make([]string, len(entries))
+	entryMap := make(map[string]*TreeEntry)
+	for i, entry := range entries {
+		entryPaths[i] = entry.Name()
+		entryMap[entry.Name()] = entry
+	}
+
+	commits, err := GetLastCommitForPaths(index, treePath, entryPaths)
+	if err != nil {
+		return err
+	}
+
+	for entry, cm := range commits {
+		if err := c.Put(index.ID().String(), path.Join(treePath, entry), cm.ID().String()); err != nil {
+			return err
+		}
+		if entryMap[entry].IsDir() {
+			subTree, err := tree.SubTree(entry)
+			if err != nil {
+				return err
+			}
+			if err := c.recursiveCache(index, subTree, entry, level-1); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}

modules/git/last_commit_cache_nogogit.go

@@ -0,0 +1,103 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build !gogit
+
+package git
+
+import (
+	"path"
+)
+
+// LastCommitCache represents a cache to store last commit
+type LastCommitCache struct {
+	repoPath    string
+	ttl         int64
+	repo        *Repository
+	commitCache map[string]*Commit
+	cache       Cache
+}
+
+// NewLastCommitCache creates a new last commit cache for repo
+func NewLastCommitCache(repoPath string, gitRepo *Repository, ttl int64, cache Cache) *LastCommitCache {
+	if cache == nil {
+		return nil
+	}
+	return &LastCommitCache{
+		repoPath:    repoPath,
+		repo:        gitRepo,
+		commitCache: make(map[string]*Commit),
+		ttl:         ttl,
+		cache:       cache,
+	}
+}
+
+// Get get the last commit information by commit id and entry path
+func (c *LastCommitCache) Get(ref, entryPath string) (interface{}, error) {
+	v := c.cache.Get(c.getCacheKey(c.repoPath, ref, entryPath))
+	if vs, ok := v.(string); ok {
+		log("LastCommitCache hit level 1: [%s:%s:%s]", ref, entryPath, vs)
+		if commit, ok := c.commitCache[vs]; ok {
+			log("LastCommitCache hit level 2: [%s:%s:%s]", ref, entryPath, vs)
+			return commit, nil
+		}
+		id, err := c.repo.ConvertToSHA1(vs)
+		if err != nil {
+			return nil, err
+		}
+		commit, err := c.repo.getCommit(id)
+		if err != nil {
+			return nil, err
+		}
+		c.commitCache[vs] = commit
+		return commit, nil
+	}
+	return nil, nil
+}
+
+// CacheCommit will cache the commit from the gitRepository
+func (c *LastCommitCache) CacheCommit(commit *Commit) error {
+	return c.recursiveCache(commit, &commit.Tree, "", 1)
+}
+
+func (c *LastCommitCache) recursiveCache(commit *Commit, tree *Tree, treePath string, level int) error {
+	if level == 0 {
+		return nil
+	}
+
+	entries, err := tree.ListEntries()
+	if err != nil {
+		return err
+	}
+
+	entryPaths := make([]string, len(entries))
+	entryMap := make(map[string]*TreeEntry)
+	for i, entry := range entries {
+		entryPaths[i] = entry.Name()
+		entryMap[entry.Name()] = entry
+	}
+
+	commits, err := GetLastCommitForPaths(commit, treePath, entryPaths)
+	if err != nil {
+		return err
+	}
+
+	for i, entryCommit := range commits {
+		entry := entryPaths[i]
+		if err := c.Put(commit.ID.String(), path.Join(treePath, entryPaths[i]), entryCommit.ID.String()); err != nil {
+			return err
+		}
+		if entryMap[entry].IsDir() {
+			subTree, err := tree.SubTree(entry)
+			if err != nil {
+				return err
+			}
+			if err := c.recursiveCache(commit, subTree, entry, level-1); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}

modules/git/notes.go

@@ -4,12 +4,6 @@
 
 package git
 
-import (
-	"io/ioutil"
-
-	"github.com/go-git/go-git/v5/plumbing/object"
-)
-
 // NotesRef is the git ref where Gitea will look for git-notes data.
 // The value ("refs/notes/commits") is the default ref used by git-notes.
 const NotesRef = "refs/notes/commits"
@@ -19,62 +13,3 @@ type Note struct {
 	Message []byte
 	Commit  *Commit
 }
-
-// GetNote retrieves the git-notes data for a given commit.
-func GetNote(repo *Repository, commitID string, note *Note) error {
-	notes, err := repo.GetCommit(NotesRef)
-	if err != nil {
-		return err
-	}
-
-	remainingCommitID := commitID
-	path := ""
-	currentTree := notes.Tree.gogitTree
-	var file *object.File
-	for len(remainingCommitID) > 2 {
-		file, err = currentTree.File(remainingCommitID)
-		if err == nil {
-			path += remainingCommitID
-			break
-		}
-		if err == object.ErrFileNotFound {
-			currentTree, err = currentTree.Tree(remainingCommitID[0:2])
-			path += remainingCommitID[0:2] + "/"
-			remainingCommitID = remainingCommitID[2:]
-		}
-		if err != nil {
-			return err
-		}
-	}
-
-	blob := file.Blob
-	dataRc, err := blob.Reader()
-	if err != nil {
-		return err
-	}
-
-	defer dataRc.Close()
-	d, err := ioutil.ReadAll(dataRc)
-	if err != nil {
-		return err
-	}
-	note.Message = d
-
-	commitNodeIndex, commitGraphFile := repo.CommitNodeIndex()
-	if commitGraphFile != nil {
-		defer commitGraphFile.Close()
-	}
-
-	commitNode, err := commitNodeIndex.Get(notes.ID)
-	if err != nil {
-		return err
-	}
-
-	lastCommits, err := GetLastCommitForPaths(commitNode, "", []string{path})
-	if err != nil {
-		return err
-	}
-	note.Commit = convertCommit(lastCommits[path])
-
-	return nil
-}

modules/git/notes_gogit.go

@@ -0,0 +1,72 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build gogit
+
+package git
+
+import (
+	"io/ioutil"
+
+	"github.com/go-git/go-git/v5/plumbing/object"
+)
+
+// GetNote retrieves the git-notes data for a given commit.
+func GetNote(repo *Repository, commitID string, note *Note) error {
+	notes, err := repo.GetCommit(NotesRef)
+	if err != nil {
+		return err
+	}
+
+	remainingCommitID := commitID
+	path := ""
+	currentTree := notes.Tree.gogitTree
+	var file *object.File
+	for len(remainingCommitID) > 2 {
+		file, err = currentTree.File(remainingCommitID)
+		if err == nil {
+			path += remainingCommitID
+			break
+		}
+		if err == object.ErrFileNotFound {
+			currentTree, err = currentTree.Tree(remainingCommitID[0:2])
+			path += remainingCommitID[0:2] + "/"
+			remainingCommitID = remainingCommitID[2:]
+		}
+		if err != nil {
+			return err
+		}
+	}
+
+	blob := file.Blob
+	dataRc, err := blob.Reader()
+	if err != nil {
+		return err
+	}
+
+	defer dataRc.Close()
+	d, err := ioutil.ReadAll(dataRc)
+	if err != nil {
+		return err
+	}
+	note.Message = d
+
+	commitNodeIndex, commitGraphFile := repo.CommitNodeIndex()
+	if commitGraphFile != nil {
+		defer commitGraphFile.Close()
+	}
+
+	commitNode, err := commitNodeIndex.Get(notes.ID)
+	if err != nil {
+		return err
+	}
+
+	lastCommits, err := GetLastCommitForPaths(commitNode, "", []string{path})
+	if err != nil {
+		return err
+	}
+	note.Commit = convertCommit(lastCommits[path])
+
+	return nil
+}

modules/git/notes_nogogit.go

@@ -0,0 +1,59 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build !gogit
+
+package git
+
+import (
+	"io/ioutil"
+)
+
+// GetNote retrieves the git-notes data for a given commit.
+func GetNote(repo *Repository, commitID string, note *Note) error {
+	notes, err := repo.GetCommit(NotesRef)
+	if err != nil {
+		return err
+	}
+
+	path := ""
+
+	tree := &notes.Tree
+
+	var entry *TreeEntry
+	for len(commitID) > 2 {
+		entry, err = tree.GetTreeEntryByPath(commitID)
+		if err == nil {
+			path += commitID
+			break
+		}
+		if IsErrNotExist(err) {
+			tree, err = tree.SubTree(commitID[0:2])
+			path += commitID[0:2] + "/"
+			commitID = commitID[2:]
+		}
+		if err != nil {
+			return err
+		}
+	}
+
+	dataRc, err := entry.Blob().DataAsync()
+	if err != nil {
+		return err
+	}
+	defer dataRc.Close()
+	d, err := ioutil.ReadAll(dataRc)
+	if err != nil {
+		return err
+	}
+	note.Message = d
+
+	lastCommits, err := GetLastCommitForPaths(notes, "", []string{path})
+	if err != nil {
+		return err
+	}
+	note.Commit = lastCommits[0]
+
+	return nil
+}

modules/git/parse_gogit.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+// +build gogit
+
 package git
 
 import (

modules/git/parse_gogit_test.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+// +build gogit
+
 package git
 
 import (

modules/git/parse_nogogit.go

@@ -0,0 +1,78 @@
+// Copyright 2018 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build !gogit
+
+package git
+
+import (
+	"bytes"
+	"fmt"
+	"strconv"
+)
+
+// ParseTreeEntries parses the output of a `git ls-tree` command.
+func ParseTreeEntries(data []byte) ([]*TreeEntry, error) {
+	return parseTreeEntries(data, nil)
+}
+
+func parseTreeEntries(data []byte, ptree *Tree) ([]*TreeEntry, error) {
+	entries := make([]*TreeEntry, 0, 10)
+	for pos := 0; pos < len(data); {
+		// expect line to be of the form "<mode> <type> <sha>\t<filename>"
+		entry := new(TreeEntry)
+		entry.ptree = ptree
+		if pos+6 > len(data) {
+			return nil, fmt.Errorf("Invalid ls-tree output: %s", string(data))
+		}
+		switch string(data[pos : pos+6]) {
+		case "100644":
+			entry.entryMode = EntryModeBlob
+			pos += 12 // skip over "100644 blob "
+		case "100755":
+			entry.entryMode = EntryModeExec
+			pos += 12 // skip over "100755 blob "
+		case "120000":
+			entry.entryMode = EntryModeSymlink
+			pos += 12 // skip over "120000 blob "
+		case "160000":
+			entry.entryMode = EntryModeCommit
+			pos += 14 // skip over "160000 object "
+		case "040000":
+			entry.entryMode = EntryModeTree
+			pos += 12 // skip over "040000 tree "
+		default:
+			return nil, fmt.Errorf("unknown type: %v", string(data[pos:pos+6]))
+		}
+
+		if pos+40 > len(data) {
+			return nil, fmt.Errorf("Invalid ls-tree output: %s", string(data))
+		}
+		id, err := NewIDFromString(string(data[pos : pos+40]))
+		if err != nil {
+			return nil, fmt.Errorf("Invalid ls-tree output: %v", err)
+		}
+		entry.ID = id
+		pos += 41 // skip over sha and trailing space
+
+		end := pos + bytes.IndexByte(data[pos:], '\n')
+		if end < pos {
+			return nil, fmt.Errorf("Invalid ls-tree output: %s", string(data))
+		}
+
+		// In case entry name is surrounded by double quotes(it happens only in git-shell).
+		if data[pos] == '"' {
+			entry.name, err = strconv.Unquote(string(data[pos:end]))
+			if err != nil {
+				return nil, fmt.Errorf("Invalid ls-tree output: %v", err)
+			}
+		} else {
+			entry.name = string(data[pos:end])
+		}
+
+		pos = end + 1
+		entries = append(entries, entry)
+	}
+	return entries, nil
+}

modules/git/pipeline/lfs.go

@@ -0,0 +1,159 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build gogit
+
+package pipeline
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"code.gitea.io/gitea/modules/git"
+	gogit "github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing/object"
+)
+
+// LFSResult represents commits found using a provided pointer file hash
+type LFSResult struct {
+	Name           string
+	SHA            string
+	Summary        string
+	When           time.Time
+	ParentHashes   []git.SHA1
+	BranchName     string
+	FullCommitName string
+}
+
+type lfsResultSlice []*LFSResult
+
+func (a lfsResultSlice) Len() int           { return len(a) }
+func (a lfsResultSlice) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a lfsResultSlice) Less(i, j int) bool { return a[j].When.After(a[i].When) }
+
+// FindLFSFile finds commits that contain a provided pointer file hash
+func FindLFSFile(repo *git.Repository, hash git.SHA1) ([]*LFSResult, error) {
+	resultsMap := map[string]*LFSResult{}
+	results := make([]*LFSResult, 0)
+
+	basePath := repo.Path
+	gogitRepo := repo.GoGitRepo()
+
+	commitsIter, err := gogitRepo.Log(&gogit.LogOptions{
+		Order: gogit.LogOrderCommitterTime,
+		All:   true,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("Failed to get GoGit CommitsIter. Error: %w", err)
+	}
+
+	err = commitsIter.ForEach(func(gitCommit *object.Commit) error {
+		tree, err := gitCommit.Tree()
+		if err != nil {
+			return err
+		}
+		treeWalker := object.NewTreeWalker(tree, true, nil)
+		defer treeWalker.Close()
+		for {
+			name, entry, err := treeWalker.Next()
+			if err == io.EOF {
+				break
+			}
+			if entry.Hash == hash {
+				result := LFSResult{
+					Name:         name,
+					SHA:          gitCommit.Hash.String(),
+					Summary:      strings.Split(strings.TrimSpace(gitCommit.Message), "\n")[0],
+					When:         gitCommit.Author.When,
+					ParentHashes: gitCommit.ParentHashes,
+				}
+				resultsMap[gitCommit.Hash.String()+":"+name] = &result
+			}
+		}
+		return nil
+	})
+	if err != nil && err != io.EOF {
+		return nil, fmt.Errorf("Failure in CommitIter.ForEach: %w", err)
+	}
+
+	for _, result := range resultsMap {
+		hasParent := false
+		for _, parentHash := range result.ParentHashes {
+			if _, hasParent = resultsMap[parentHash.String()+":"+result.Name]; hasParent {
+				break
+			}
+		}
+		if !hasParent {
+			results = append(results, result)
+		}
+	}
+
+	sort.Sort(lfsResultSlice(results))
+
+	// Should really use a go-git function here but name-rev is not completed and recapitulating it is not simple
+	shasToNameReader, shasToNameWriter := io.Pipe()
+	nameRevStdinReader, nameRevStdinWriter := io.Pipe()
+	errChan := make(chan error, 1)
+	wg := sync.WaitGroup{}
+	wg.Add(3)
+
+	go func() {
+		defer wg.Done()
+		scanner := bufio.NewScanner(nameRevStdinReader)
+		i := 0
+		for scanner.Scan() {
+			line := scanner.Text()
+			if len(line) == 0 {
+				continue
+			}
+			result := results[i]
+			result.FullCommitName = line
+			result.BranchName = strings.Split(line, "~")[0]
+			i++
+		}
+	}()
+	go NameRevStdin(shasToNameReader, nameRevStdinWriter, &wg, basePath)
+	go func() {
+		defer wg.Done()
+		defer shasToNameWriter.Close()
+		for _, result := range results {
+			i := 0
+			if i < len(result.SHA) {
+				n, err := shasToNameWriter.Write([]byte(result.SHA)[i:])
+				if err != nil {
+					errChan <- err
+					break
+				}
+				i += n
+			}
+			n := 0
+			for n < 1 {
+				n, err = shasToNameWriter.Write([]byte{'\n'})
+				if err != nil {
+					errChan <- err
+					break
+				}
+
+			}
+
+		}
+	}()
+
+	wg.Wait()
+
+	select {
+	case err, has := <-errChan:
+		if has {
+			return nil, fmt.Errorf("Unable to obtain name for LFS files. Error: %w", err)
+		}
+	default:
+	}
+
+	return results, nil
+}

modules/git/pipeline/lfs_nogogit.go

@@ -0,0 +1,266 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build !gogit
+
+package pipeline
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"code.gitea.io/gitea/modules/git"
+)
+
+// LFSResult represents commits found using a provided pointer file hash
+type LFSResult struct {
+	Name           string
+	SHA            string
+	Summary        string
+	When           time.Time
+	ParentHashes   []git.SHA1
+	BranchName     string
+	FullCommitName string
+}
+
+type lfsResultSlice []*LFSResult
+
+func (a lfsResultSlice) Len() int           { return len(a) }
+func (a lfsResultSlice) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a lfsResultSlice) Less(i, j int) bool { return a[j].When.After(a[i].When) }
+
+// FindLFSFile finds commits that contain a provided pointer file hash
+func FindLFSFile(repo *git.Repository, hash git.SHA1) ([]*LFSResult, error) {
+	resultsMap := map[string]*LFSResult{}
+	results := make([]*LFSResult, 0)
+
+	basePath := repo.Path
+
+	hashStr := hash.String()
+
+	// Use rev-list to provide us with all commits in order
+	revListReader, revListWriter := io.Pipe()
+	defer func() {
+		_ = revListWriter.Close()
+		_ = revListReader.Close()
+	}()
+
+	go func() {
+		stderr := strings.Builder{}
+		err := git.NewCommand("rev-list", "--all").RunInDirPipeline(repo.Path, revListWriter, &stderr)
+		if err != nil {
+			_ = revListWriter.CloseWithError(git.ConcatenateError(err, (&stderr).String()))
+		} else {
+			_ = revListWriter.Close()
+		}
+	}()
+
+	// Next feed the commits in order into cat-file --batch, followed by their trees and sub trees as necessary.
+	// so let's create a batch stdin and stdout
+	batchStdinReader, batchStdinWriter := io.Pipe()
+	batchStdoutReader, batchStdoutWriter := io.Pipe()
+	defer func() {
+		_ = batchStdinReader.Close()
+		_ = batchStdinWriter.Close()
+		_ = batchStdoutReader.Close()
+		_ = batchStdoutWriter.Close()
+	}()
+
+	go func() {
+		stderr := strings.Builder{}
+		err := git.NewCommand("cat-file", "--batch").RunInDirFullPipeline(repo.Path, batchStdoutWriter, &stderr, batchStdinReader)
+		if err != nil {
+			_ = revListWriter.CloseWithError(git.ConcatenateError(err, (&stderr).String()))
+		} else {
+			_ = revListWriter.Close()
+		}
+	}()
+
+	// For simplicities sake we'll us a buffered reader to read from the cat-file --batch
+	batchReader := bufio.NewReader(batchStdoutReader)
+
+	// We'll use a scanner for the revList because it's simpler than a bufio.Reader
+	scan := bufio.NewScanner(revListReader)
+	trees := [][]byte{}
+	paths := []string{}
+
+	fnameBuf := make([]byte, 4096)
+	modeBuf := make([]byte, 40)
+	workingShaBuf := make([]byte, 40)
+
+	for scan.Scan() {
+		// Get the next commit ID
+		commitID := scan.Bytes()
+
+		// push the commit to the cat-file --batch process
+		_, err := batchStdinWriter.Write(commitID)
+		if err != nil {
+			return nil, err
+		}
+		_, err = batchStdinWriter.Write([]byte{'\n'})
+		if err != nil {
+			return nil, err
+		}
+
+		var curCommit *git.Commit
+		curPath := ""
+
+	commitReadingLoop:
+		for {
+			_, typ, size, err := git.ReadBatchLine(batchReader)
+			if err != nil {
+				return nil, err
+			}
+
+			switch typ {
+			case "tag":
+				// This shouldn't happen but if it does well just get the commit and try again
+				id, err := git.ReadTagObjectID(batchReader, size)
+				if err != nil {
+					return nil, err
+				}
+				_, err = batchStdinWriter.Write([]byte(id + "\n"))
+				if err != nil {
+					return nil, err
+				}
+				continue
+			case "commit":
+				// Read in the commit to get its tree and in case this is one of the last used commits
+				curCommit, err = git.CommitFromReader(repo, git.MustIDFromString(string(commitID)), io.LimitReader(batchReader, int64(size)))
+				if err != nil {
+					return nil, err
+				}
+
+				_, err := batchStdinWriter.Write([]byte(curCommit.Tree.ID.String() + "\n"))
+				if err != nil {
+					return nil, err
+				}
+				curPath = ""
+			case "tree":
+				var n int64
+				for n < size {
+					mode, fname, sha, count, err := git.ParseTreeLine(batchReader, modeBuf, fnameBuf, workingShaBuf)
+					if err != nil {
+						return nil, err
+					}
+					n += int64(count)
+					if bytes.Equal(sha, []byte(hashStr)) {
+						result := LFSResult{
+							Name:         curPath + string(fname),
+							SHA:          curCommit.ID.String(),
+							Summary:      strings.Split(strings.TrimSpace(curCommit.CommitMessage), "\n")[0],
+							When:         curCommit.Author.When,
+							ParentHashes: curCommit.Parents,
+						}
+						resultsMap[curCommit.ID.String()+":"+curPath+string(fname)] = &result
+					} else if string(mode) == git.EntryModeTree.String() {
+						trees = append(trees, sha)
+						paths = append(paths, curPath+string(fname)+"/")
+					}
+				}
+				if len(trees) > 0 {
+					_, err := batchStdinWriter.Write(trees[len(trees)-1])
+					if err != nil {
+						return nil, err
+					}
+					_, err = batchStdinWriter.Write([]byte("\n"))
+					if err != nil {
+						return nil, err
+					}
+					curPath = paths[len(paths)-1]
+					trees = trees[:len(trees)-1]
+					paths = paths[:len(paths)-1]
+				} else {
+					break commitReadingLoop
+				}
+			}
+		}
+	}
+
+	if err := scan.Err(); err != nil {
+		return nil, err
+	}
+
+	for _, result := range resultsMap {
+		hasParent := false
+		for _, parentHash := range result.ParentHashes {
+			if _, hasParent = resultsMap[parentHash.String()+":"+result.Name]; hasParent {
+				break
+			}
+		}
+		if !hasParent {
+			results = append(results, result)
+		}
+	}
+
+	sort.Sort(lfsResultSlice(results))
+
+	// Should really use a go-git function here but name-rev is not completed and recapitulating it is not simple
+	shasToNameReader, shasToNameWriter := io.Pipe()
+	nameRevStdinReader, nameRevStdinWriter := io.Pipe()
+	errChan := make(chan error, 1)
+	wg := sync.WaitGroup{}
+	wg.Add(3)
+
+	go func() {
+		defer wg.Done()
+		scanner := bufio.NewScanner(nameRevStdinReader)
+		i := 0
+		for scanner.Scan() {
+			line := scanner.Text()
+			if len(line) == 0 {
+				continue
+			}
+			result := results[i]
+			result.FullCommitName = line
+			result.BranchName = strings.Split(line, "~")[0]
+			i++
+		}
+	}()
+	go NameRevStdin(shasToNameReader, nameRevStdinWriter, &wg, basePath)
+	go func() {
+		defer wg.Done()
+		defer shasToNameWriter.Close()
+		for _, result := range results {
+			i := 0
+			if i < len(result.SHA) {
+				n, err := shasToNameWriter.Write([]byte(result.SHA)[i:])
+				if err != nil {
+					errChan <- err
+					break
+				}
+				i += n
+			}
+			var err error
+			n := 0
+			for n < 1 {
+				n, err = shasToNameWriter.Write([]byte{'\n'})
+				if err != nil {
+					errChan <- err
+					break
+				}
+
+			}
+
+		}
+	}()
+
+	wg.Wait()
+
+	select {
+	case err, has := <-errChan:
+		if has {
+			return nil, fmt.Errorf("Unable to obtain name for LFS files. Error: %w", err)
+		}
+	default:
+	}
+
+	return results, nil
+}

modules/git/repo.go

@@ -9,34 +9,16 @@ import (
 	"bytes"
 	"container/list"
 	"context"
-	"errors"
 	"fmt"
 	"os"
 	"path"
-	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
 
-	gitealog "code.gitea.io/gitea/modules/log"
-	"github.com/go-git/go-billy/v5/osfs"
-	gogit "github.com/go-git/go-git/v5"
-	"github.com/go-git/go-git/v5/plumbing/cache"
-	"github.com/go-git/go-git/v5/storage/filesystem"
 	"github.com/unknwon/com"
 )
 
-// Repository represents a Git repository.
-type Repository struct {
-	Path string
-
-	tagCache *ObjectCache
-
-	gogitRepo    *gogit.Repository
-	gogitStorage *filesystem.Storage
-	gpgSettings  *GPGSettings
-}
-
 // GPGSettings represents the default GPG settings for this repository
 type GPGSettings struct {
 	Sign             bool
@@ -93,52 +75,6 @@ func InitRepository(repoPath string, bare bool) error {
 	return err
 }
 
-// OpenRepository opens the repository at the given path.
-func OpenRepository(repoPath string) (*Repository, error) {
-	repoPath, err := filepath.Abs(repoPath)
-	if err != nil {
-		return nil, err
-	} else if !isDir(repoPath) {
-		return nil, errors.New("no such file or directory")
-	}
-
-	fs := osfs.New(repoPath)
-	_, err = fs.Stat(".git")
-	if err == nil {
-		fs, err = fs.Chroot(".git")
-		if err != nil {
-			return nil, err
-		}
-	}
-	storage := filesystem.NewStorageWithOptions(fs, cache.NewObjectLRUDefault(), filesystem.Options{KeepDescriptors: true})
-	gogitRepo, err := gogit.Open(storage, fs)
-	if err != nil {
-		return nil, err
-	}
-
-	return &Repository{
-		Path:         repoPath,
-		gogitRepo:    gogitRepo,
-		gogitStorage: storage,
-		tagCache:     newObjectCache(),
-	}, nil
-}
-
-// Close this repository, in particular close the underlying gogitStorage if this is not nil
-func (repo *Repository) Close() {
-	if repo == nil || repo.gogitStorage == nil {
-		return
-	}
-	if err := repo.gogitStorage.Close(); err != nil {
-		gitealog.Error("Error closing storage: %v", err)
-	}
-}
-
-// GoGitRepo gets the go-git repo representation
-func (repo *Repository) GoGitRepo() *gogit.Repository {
-	return repo.gogitRepo
-}
-
 // IsEmpty Check if repository is empty.
 func (repo *Repository) IsEmpty() (bool, error) {
 	var errbuf strings.Builder

modules/git/repo_base_gogit.go

@@ -0,0 +1,76 @@
+// Copyright 2015 The Gogs Authors. All rights reserved.
+// Copyright 2017 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build gogit
+
+package git
+
+import (
+	"errors"
+	"path/filepath"
+
+	gitealog "code.gitea.io/gitea/modules/log"
+	"github.com/go-git/go-billy/v5/osfs"
+	gogit "github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing/cache"
+	"github.com/go-git/go-git/v5/storage/filesystem"
+)
+
+// Repository represents a Git repository.
+type Repository struct {
+	Path string
+
+	tagCache *ObjectCache
+
+	gogitRepo    *gogit.Repository
+	gogitStorage *filesystem.Storage
+	gpgSettings  *GPGSettings
+}
+
+// OpenRepository opens the repository at the given path.
+func OpenRepository(repoPath string) (*Repository, error) {
+	repoPath, err := filepath.Abs(repoPath)
+	if err != nil {
+		return nil, err
+	} else if !isDir(repoPath) {
+		return nil, errors.New("no such file or directory")
+	}
+
+	fs := osfs.New(repoPath)
+	_, err = fs.Stat(".git")
+	if err == nil {
+		fs, err = fs.Chroot(".git")
+		if err != nil {
+			return nil, err
+		}
+	}
+	storage := filesystem.NewStorageWithOptions(fs, cache.NewObjectLRUDefault(), filesystem.Options{KeepDescriptors: true})
+	gogitRepo, err := gogit.Open(storage, fs)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Repository{
+		Path:         repoPath,
+		gogitRepo:    gogitRepo,
+		gogitStorage: storage,
+		tagCache:     newObjectCache(),
+	}, nil
+}
+
+// Close this repository, in particular close the underlying gogitStorage if this is not nil
+func (repo *Repository) Close() {
+	if repo == nil || repo.gogitStorage == nil {
+		return
+	}
+	if err := repo.gogitStorage.Close(); err != nil {
+		gitealog.Error("Error closing storage: %v", err)
+	}
+}
+
+// GoGitRepo gets the go-git repo representation
+func (repo *Repository) GoGitRepo() *gogit.Repository {
+	return repo.gogitRepo
+}

modules/git/repo_base_nogogit.go

@@ -0,0 +1,40 @@
+// Copyright 2015 The Gogs Authors. All rights reserved.
+// Copyright 2017 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build !gogit
+
+package git
+
+import (
+	"errors"
+	"path/filepath"
+)
+
+// Repository represents a Git repository.
+type Repository struct {
+	Path string
+
+	tagCache *ObjectCache
+
+	gpgSettings *GPGSettings
+}
+
+// OpenRepository opens the repository at the given path.
+func OpenRepository(repoPath string) (*Repository, error) {
+	repoPath, err := filepath.Abs(repoPath)
+	if err != nil {
+		return nil, err
+	} else if !isDir(repoPath) {
+		return nil, errors.New("no such file or directory")
+	}
+	return &Repository{
+		Path:     repoPath,
+		tagCache: newObjectCache(),
+	}, nil
+}
+
+// Close this repository, in particular close the underlying gogitStorage if this is not nil
+func (repo *Repository) Close() {
+}

modules/git/repo_blob.go

@@ -1,25 +1,9 @@
-// Copyright 2018 The Gitea Authors. All rights reserved.
+// Copyright 2020 The Gitea Authors. All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
 package git
 
-import (
-	"github.com/go-git/go-git/v5/plumbing"
-)
-
-func (repo *Repository) getBlob(id SHA1) (*Blob, error) {
-	encodedObj, err := repo.gogitRepo.Storer.EncodedObject(plumbing.AnyObject, id)
-	if err != nil {
-		return nil, ErrNotExist{id.String(), ""}
-	}
-
-	return &Blob{
-		ID:              id,
-		gogitEncodedObj: encodedObj,
-	}, nil
-}
-
 // GetBlob finds the blob object in the repository.
 func (repo *Repository) GetBlob(idStr string) (*Blob, error) {
 	id, err := NewIDFromString(idStr)

modules/git/repo_blob_gogit.go

@@ -0,0 +1,23 @@
+// Copyright 2018 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build gogit
+
+package git
+
+import (
+	"github.com/go-git/go-git/v5/plumbing"
+)
+
+func (repo *Repository) getBlob(id SHA1) (*Blob, error) {
+	encodedObj, err := repo.gogitRepo.Storer.EncodedObject(plumbing.AnyObject, id)
+	if err != nil {
+		return nil, ErrNotExist{id.String(), ""}
+	}
+
+	return &Blob{
+		ID:              id,
+		gogitEncodedObj: encodedObj,
+	}, nil
+}

modules/git/repo_blob_nogogit.go

@@ -0,0 +1,17 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build !gogit
+
+package git
+
+func (repo *Repository) getBlob(id SHA1) (*Blob, error) {
+	if id.IsZero() {
+		return nil, ErrNotExist{id.String(), ""}
+	}
+	return &Blob{
+		ID:       id,
+		repoPath: repo.Path,
+	}, nil
+}

modules/git/repo_branch.go

@@ -8,8 +8,6 @@ package git
 import (
 	"fmt"
 	"strings"
-
-	"github.com/go-git/go-git/v5/plumbing"
 )
 
 // BranchPrefix base dir of the branch information file store on git
@@ -26,18 +24,6 @@ func IsBranchExist(repoPath, name string) bool {
 	return IsReferenceExist(repoPath, BranchPrefix+name)
 }
 
-// IsBranchExist returns true if given branch exists in current repository.
-func (repo *Repository) IsBranchExist(name string) bool {
-	if name == "" {
-		return false
-	}
-	reference, err := repo.gogitRepo.Reference(plumbing.ReferenceName(BranchPrefix+name), true)
-	if err != nil {
-		return false
-	}
-	return reference.Type() != plumbing.InvalidReference
-}
-
 // Branch represents a Git branch.
 type Branch struct {
 	Name string
@@ -79,25 +65,6 @@ func (repo *Repository) GetDefaultBranch() (string, error) {
 	return NewCommand("symbolic-ref", "HEAD").RunInDir(repo.Path)
 }
 
-// GetBranches returns all branches of the repository.
-func (repo *Repository) GetBranches() ([]string, error) {
-	var branchNames []string
-
-	branches, err := repo.gogitRepo.Branches()
-	if err != nil {
-		return nil, err
-	}
-
-	_ = branches.ForEach(func(branch *plumbing.Reference) error {
-		branchNames = append(branchNames, strings.TrimPrefix(branch.Name().String(), BranchPrefix))
-		return nil
-	})
-
-	// TODO: Sort?
-
-	return branchNames, nil
-}
-
 // GetBranch returns a branch by it's name
 func (repo *Repository) GetBranch(branch string) (*Branch, error) {
 	if !repo.IsBranchExist(branch) {

modules/git/repo_branch_gogit.go

@@ -0,0 +1,45 @@
+// Copyright 2015 The Gogs Authors. All rights reserved.
+// Copyright 2018 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build gogit
+
+package git
+
+import (
+	"strings"
+
+	"github.com/go-git/go-git/v5/plumbing"
+)
+
+// IsBranchExist returns true if given branch exists in current repository.
+func (repo *Repository) IsBranchExist(name string) bool {
+	if name == "" {
+		return false
+	}
+	reference, err := repo.gogitRepo.Reference(plumbing.ReferenceName(BranchPrefix+name), true)
+	if err != nil {
+		return false
+	}
+	return reference.Type() != plumbing.InvalidReference
+}
+
+// GetBranches returns all branches of the repository.
+func (repo *Repository) GetBranches() ([]string, error) {
+	var branchNames []string
+
+	branches, err := repo.gogitRepo.Branches()
+	if err != nil {
+		return nil, err
+	}
+
+	_ = branches.ForEach(func(branch *plumbing.Reference) error {
+		branchNames = append(branchNames, strings.TrimPrefix(branch.Name().String(), BranchPrefix))
+		return nil
+	})
+
+	// TODO: Sort?
+
+	return branchNames, nil
+}

modules/git/repo_branch_nogogit.go

@@ -0,0 +1,82 @@
+// Copyright 2015 The Gogs Authors. All rights reserved.
+// Copyright 2018 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build !gogit
+
+package git
+
+import (
+	"bufio"
+	"io"
+	"strings"
+)
+
+// IsBranchExist returns true if given branch exists in current repository.
+func (repo *Repository) IsBranchExist(name string) bool {
+	if name == "" {
+		return false
+	}
+	return IsReferenceExist(repo.Path, BranchPrefix+name)
+}
+
+// GetBranches returns all branches of the repository.
+func (repo *Repository) GetBranches() ([]string, error) {
+	return callShowRef(repo.Path, BranchPrefix, "--heads")
+}
+
+func callShowRef(repoPath, prefix, arg string) ([]string, error) {
+	var branchNames []string
+
+	stdoutReader, stdoutWriter := io.Pipe()
+	defer func() {
+		_ = stdoutReader.Close()
+		_ = stdoutWriter.Close()
+	}()
+
+	go func() {
+		stderrBuilder := &strings.Builder{}
+		err := NewCommand("show-ref", arg).RunInDirPipeline(repoPath, stdoutWriter, stderrBuilder)
+		if err != nil {
+			if stderrBuilder.Len() == 0 {
+				_ = stdoutWriter.Close()
+				return
+			}
+			_ = stdoutWriter.CloseWithError(ConcatenateError(err, stderrBuilder.String()))
+		} else {
+			_ = stdoutWriter.Close()
+		}
+	}()
+
+	bufReader := bufio.NewReader(stdoutReader)
+	for {
+		// The output of show-ref is simply a list:
+		// <sha> SP <ref> LF
+		_, err := bufReader.ReadSlice(' ')
+		for err == bufio.ErrBufferFull {
+			// This shouldn't happen but we'll tolerate it for the sake of peace
+			_, err = bufReader.ReadSlice(' ')
+		}
+		if err == io.EOF {
+			return branchNames, nil
+		}
+		if err != nil {
+			return nil, err
+		}
+
+		branchName, err := bufReader.ReadString('\n')
+		if err == io.EOF {
+			// This shouldn't happen... but we'll tolerate it for the sake of peace
+			return branchNames, nil
+		}
+		if err != nil {
+			return nil, err
+		}
+		branchName = strings.TrimPrefix(branchName, prefix)
+		if len(branchName) > 0 {
+			branchName = branchName[:len(branchName)-1]
+		}
+		branchNames = append(branchNames, branchName)
+	}
+}

modules/git/repo_commit.go

@@ -8,36 +8,10 @@ package git
 import (
 	"bytes"
 	"container/list"
-	"fmt"
 	"strconv"
 	"strings"
-
-	"github.com/go-git/go-git/v5/plumbing"
-	"github.com/go-git/go-git/v5/plumbing/object"
 )
 
-// GetRefCommitID returns the last commit ID string of given reference (branch or tag).
-func (repo *Repository) GetRefCommitID(name string) (string, error) {
-	ref, err := repo.gogitRepo.Reference(plumbing.ReferenceName(name), true)
-	if err != nil {
-		if err == plumbing.ErrReferenceNotFound {
-			return "", ErrNotExist{
-				ID: name,
-			}
-		}
-		return "", err
-	}
-
-	return ref.Hash().String(), nil
-}
-
-// IsCommitExist returns true if given commit exists in current repository.
-func (repo *Repository) IsCommitExist(name string) bool {
-	hash := plumbing.NewHash(name)
-	_, err := repo.gogitRepo.CommitObject(hash)
-	return err == nil
-}
-
 // GetBranchCommitID returns last commit ID string of given branch.
 func (repo *Repository) GetBranchCommitID(name string) (string, error) {
 	return repo.GetRefCommitID(BranchPrefix + name)
@@ -55,78 +29,6 @@ func (repo *Repository) GetTagCommitID(name string) (string, error) {
 	return strings.TrimSpace(stdout), nil
 }
 
-func convertPGPSignatureForTag(t *object.Tag) *CommitGPGSignature {
-	if t.PGPSignature == "" {
-		return nil
-	}
-
-	var w strings.Builder
-	var err error
-
-	if _, err = fmt.Fprintf(&w,
-		"object %s\ntype %s\ntag %s\ntagger ",
-		t.Target.String(), t.TargetType.Bytes(), t.Name); err != nil {
-		return nil
-	}
-
-	if err = t.Tagger.Encode(&w); err != nil {
-		return nil
-	}
-
-	if _, err = fmt.Fprintf(&w, "\n\n"); err != nil {
-		return nil
-	}
-
-	if _, err = fmt.Fprintf(&w, t.Message); err != nil {
-		return nil
-	}
-
-	return &CommitGPGSignature{
-		Signature: t.PGPSignature,
-		Payload:   strings.TrimSpace(w.String()) + "\n",
-	}
-}
-
-func (repo *Repository) getCommit(id SHA1) (*Commit, error) {
-	var tagObject *object.Tag
-
-	gogitCommit, err := repo.gogitRepo.CommitObject(id)
-	if err == plumbing.ErrObjectNotFound {
-		tagObject, err = repo.gogitRepo.TagObject(id)
-		if err == plumbing.ErrObjectNotFound {
-			return nil, ErrNotExist{
-				ID: id.String(),
-			}
-		}
-		if err == nil {
-			gogitCommit, err = repo.gogitRepo.CommitObject(tagObject.Target)
-		}
-		// if we get a plumbing.ErrObjectNotFound here then the repository is broken and it should be 500
-	}
-	if err != nil {
-		return nil, err
-	}
-
-	commit := convertCommit(gogitCommit)
-	commit.repo = repo
-
-	if tagObject != nil {
-		commit.CommitMessage = strings.TrimSpace(tagObject.Message)
-		commit.Author = &tagObject.Tagger
-		commit.Signature = convertPGPSignatureForTag(tagObject)
-	}
-
-	tree, err := gogitCommit.Tree()
-	if err != nil {
-		return nil, err
-	}
-
-	commit.Tree.ID = tree.Hash
-	commit.Tree.gogitTree = tree
-
-	return commit, nil
-}
-
 // ConvertToSHA1 returns a Hash object from a potential ID string
 func (repo *Repository) ConvertToSHA1(commitID string) (SHA1, error) {
 	if len(commitID) != 40 {

modules/git/repo_commit_gogit.go

@@ -0,0 +1,110 @@
+// Copyright 2015 The Gogs Authors. All rights reserved.
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build gogit
+
+package git
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/go-git/go-git/v5/plumbing/object"
+)
+
+// GetRefCommitID returns the last commit ID string of given reference (branch or tag).
+func (repo *Repository) GetRefCommitID(name string) (string, error) {
+	ref, err := repo.gogitRepo.Reference(plumbing.ReferenceName(name), true)
+	if err != nil {
+		if err == plumbing.ErrReferenceNotFound {
+			return "", ErrNotExist{
+				ID: name,
+			}
+		}
+		return "", err
+	}
+
+	return ref.Hash().String(), nil
+}
+
+// IsCommitExist returns true if given commit exists in current repository.
+func (repo *Repository) IsCommitExist(name string) bool {
+	hash := plumbing.NewHash(name)
+	_, err := repo.gogitRepo.CommitObject(hash)
+	return err == nil
+}
+
+func convertPGPSignatureForTag(t *object.Tag) *CommitGPGSignature {
+	if t.PGPSignature == "" {
+		return nil
+	}
+
+	var w strings.Builder
+	var err error
+
+	if _, err = fmt.Fprintf(&w,
+		"object %s\ntype %s\ntag %s\ntagger ",
+		t.Target.String(), t.TargetType.Bytes(), t.Name); err != nil {
+		return nil
+	}
+
+	if err = t.Tagger.Encode(&w); err != nil {
+		return nil
+	}
+
+	if _, err = fmt.Fprintf(&w, "\n\n"); err != nil {
+		return nil
+	}
+
+	if _, err = fmt.Fprintf(&w, t.Message); err != nil {
+		return nil
+	}
+
+	return &CommitGPGSignature{
+		Signature: t.PGPSignature,
+		Payload:   strings.TrimSpace(w.String()) + "\n",
+	}
+}
+
+func (repo *Repository) getCommit(id SHA1) (*Commit, error) {
+	var tagObject *object.Tag
+
+	gogitCommit, err := repo.gogitRepo.CommitObject(id)
+	if err == plumbing.ErrObjectNotFound {
+		tagObject, err = repo.gogitRepo.TagObject(id)
+		if err == plumbing.ErrObjectNotFound {
+			return nil, ErrNotExist{
+				ID: id.String(),
+			}
+		}
+		if err == nil {
+			gogitCommit, err = repo.gogitRepo.CommitObject(tagObject.Target)
+		}
+		// if we get a plumbing.ErrObjectNotFound here then the repository is broken and it should be 500
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	commit := convertCommit(gogitCommit)
+	commit.repo = repo
+
+	if tagObject != nil {
+		commit.CommitMessage = strings.TrimSpace(tagObject.Message)
+		commit.Author = &tagObject.Tagger
+		commit.Signature = convertPGPSignatureForTag(tagObject)
+	}
+
+	tree, err := gogitCommit.Tree()
+	if err != nil {
+		return nil, err
+	}
+
+	commit.Tree.ID = tree.Hash
+	commit.Tree.gogitTree = tree
+
+	return commit, nil
+}

modules/git/repo_commit_nogogit.go

@@ -0,0 +1,109 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build !gogit
+
+package git
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"strings"
+)
+
+// ResolveReference resolves a name to a reference
+func (repo *Repository) ResolveReference(name string) (string, error) {
+	stdout, err := NewCommand("show-ref", "--hash", name).RunInDir(repo.Path)
+	if err != nil {
+		if strings.Contains(err.Error(), "not a valid ref") {
+			return "", ErrNotExist{name, ""}
+		}
+		return "", err
+	}
+	stdout = strings.TrimSpace(stdout)
+	if stdout == "" {
+		return "", ErrNotExist{name, ""}
+	}
+
+	return stdout, nil
+}
+
+// GetRefCommitID returns the last commit ID string of given reference (branch or tag).
+func (repo *Repository) GetRefCommitID(name string) (string, error) {
+	stdout, err := NewCommand("show-ref", "--verify", "--hash", name).RunInDir(repo.Path)
+	if err != nil {
+		if strings.Contains(err.Error(), "not a valid ref") {
+			return "", ErrNotExist{name, ""}
+		}
+		return "", err
+	}
+
+	return strings.TrimSpace(stdout), nil
+}
+
+// IsCommitExist returns true if given commit exists in current repository.
+func (repo *Repository) IsCommitExist(name string) bool {
+	_, err := NewCommand("cat-file", "-e", name).RunInDir(repo.Path)
+	return err == nil
+}
+
+func (repo *Repository) getCommit(id SHA1) (*Commit, error) {
+	stdoutReader, stdoutWriter := io.Pipe()
+	defer func() {
+		_ = stdoutReader.Close()
+		_ = stdoutWriter.Close()
+	}()
+
+	go func() {
+		stderr := strings.Builder{}
+		err := NewCommand("cat-file", "--batch").RunInDirFullPipeline(repo.Path, stdoutWriter, &stderr, strings.NewReader(id.String()+"\n"))
+		if err != nil {
+			_ = stdoutWriter.CloseWithError(ConcatenateError(err, (&stderr).String()))
+		} else {
+			_ = stdoutWriter.Close()
+		}
+	}()
+
+	bufReader := bufio.NewReader(stdoutReader)
+	_, typ, size, err := ReadBatchLine(bufReader)
+	if err != nil {
+		return nil, err
+	}
+
+	switch typ {
+	case "tag":
+		// then we need to parse the tag
+		// and load the commit
+		data, err := ioutil.ReadAll(io.LimitReader(bufReader, size))
+		if err != nil {
+			return nil, err
+		}
+		tag, err := parseTagData(data)
+		if err != nil {
+			return nil, err
+		}
+		tag.repo = repo
+
+		commit, err := tag.Commit()
+		if err != nil {
+			return nil, err
+		}
+
+		commit.CommitMessage = strings.TrimSpace(tag.Message)
+		commit.Author = tag.Tagger
+		commit.Signature = tag.Signature
+
+		return commit, nil
+	case "commit":
+		return CommitFromReader(repo, id, io.LimitReader(bufReader, size))
+	default:
+		_ = stdoutReader.CloseWithError(fmt.Errorf("unknown typ: %s", typ))
+		log("Unknown typ: %s", typ)
+		return nil, ErrNotExist{
+			ID: id.String(),
+		}
+	}
+}

modules/git/repo_commitgraph_gogit.go

@@ -3,6 +3,8 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+// +build gogit
+
 package git
 
 import (

modules/git/repo_language_stats.go

@@ -4,111 +4,5 @@
 
 package git
 
-import (
-	"bytes"
-	"io"
-	"io/ioutil"
-
-	"code.gitea.io/gitea/modules/analyze"
-
-	"github.com/go-enry/go-enry/v2"
-	"github.com/go-git/go-git/v5"
-	"github.com/go-git/go-git/v5/plumbing"
-	"github.com/go-git/go-git/v5/plumbing/object"
-)
-
 const fileSizeLimit int64 = 16 * 1024 // 16 KiB
 const bigFileSize int64 = 1024 * 1024 // 1 MiB
-
-// GetLanguageStats calculates language stats for git repository at specified commit
-func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, error) {
-	r, err := git.PlainOpen(repo.Path)
-	if err != nil {
-		return nil, err
-	}
-
-	rev, err := r.ResolveRevision(plumbing.Revision(commitID))
-	if err != nil {
-		return nil, err
-	}
-
-	commit, err := r.CommitObject(*rev)
-	if err != nil {
-		return nil, err
-	}
-
-	tree, err := commit.Tree()
-	if err != nil {
-		return nil, err
-	}
-
-	sizes := make(map[string]int64)
-	err = tree.Files().ForEach(func(f *object.File) error {
-		if f.Size == 0 || enry.IsVendor(f.Name) || enry.IsDotFile(f.Name) ||
-			enry.IsDocumentation(f.Name) || enry.IsConfiguration(f.Name) {
-			return nil
-		}
-
-		// If content can not be read or file is too big just do detection by filename
-		var content []byte
-		if f.Size <= bigFileSize {
-			content, _ = readFile(f, fileSizeLimit)
-		}
-		if enry.IsGenerated(f.Name, content) {
-			return nil
-		}
-
-		// TODO: Use .gitattributes file for linguist overrides
-
-		language := analyze.GetCodeLanguage(f.Name, content)
-		if language == enry.OtherLanguage || language == "" {
-			return nil
-		}
-
-		// group languages, such as Pug -> HTML; SCSS -> CSS
-		group := enry.GetLanguageGroup(language)
-		if group != "" {
-			language = group
-		}
-
-		sizes[language] += f.Size
-
-		return nil
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	// filter special languages unless they are the only language
-	if len(sizes) > 1 {
-		for language := range sizes {
-			langtype := enry.GetLanguageType(language)
-			if langtype != enry.Programming && langtype != enry.Markup {
-				delete(sizes, language)
-			}
-		}
-	}
-
-	return sizes, nil
-}
-
-func readFile(f *object.File, limit int64) ([]byte, error) {
-	r, err := f.Reader()
-	if err != nil {
-		return nil, err
-	}
-	defer r.Close()
-
-	if limit <= 0 {
-		return ioutil.ReadAll(r)
-	}
-
-	size := f.Size
-	if limit > 0 && size > limit {
-		size = limit
-	}
-	buf := bytes.NewBuffer(nil)
-	buf.Grow(int(size))
-	_, err = io.Copy(buf, io.LimitReader(r, limit))
-	return buf.Bytes(), err
-}

modules/git/repo_language_stats_gogit.go

@@ -0,0 +1,113 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build gogit
+
+package git
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+
+	"code.gitea.io/gitea/modules/analyze"
+
+	"github.com/go-enry/go-enry/v2"
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/go-git/go-git/v5/plumbing/object"
+)
+
+// GetLanguageStats calculates language stats for git repository at specified commit
+func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, error) {
+	r, err := git.PlainOpen(repo.Path)
+	if err != nil {
+		return nil, err
+	}
+
+	rev, err := r.ResolveRevision(plumbing.Revision(commitID))
+	if err != nil {
+		return nil, err
+	}
+
+	commit, err := r.CommitObject(*rev)
+	if err != nil {
+		return nil, err
+	}
+
+	tree, err := commit.Tree()
+	if err != nil {
+		return nil, err
+	}
+
+	sizes := make(map[string]int64)
+	err = tree.Files().ForEach(func(f *object.File) error {
+		if f.Size == 0 || enry.IsVendor(f.Name) || enry.IsDotFile(f.Name) ||
+			enry.IsDocumentation(f.Name) || enry.IsConfiguration(f.Name) {
+			return nil
+		}
+
+		// If content can not be read or file is too big just do detection by filename
+		var content []byte
+		if f.Size <= bigFileSize {
+			content, _ = readFile(f, fileSizeLimit)
+		}
+		if enry.IsGenerated(f.Name, content) {
+			return nil
+		}
+
+		// TODO: Use .gitattributes file for linguist overrides
+
+		language := analyze.GetCodeLanguage(f.Name, content)
+		if language == enry.OtherLanguage || language == "" {
+			return nil
+		}
+
+		// group languages, such as Pug -> HTML; SCSS -> CSS
+		group := enry.GetLanguageGroup(language)
+		if group != "" {
+			language = group
+		}
+
+		sizes[language] += f.Size
+
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	// filter special languages unless they are the only language
+	if len(sizes) > 1 {
+		for language := range sizes {
+			langtype := enry.GetLanguageType(language)
+			if langtype != enry.Programming && langtype != enry.Markup {
+				delete(sizes, language)
+			}
+		}
+	}
+
+	return sizes, nil
+}
+
+func readFile(f *object.File, limit int64) ([]byte, error) {
+	r, err := f.Reader()
+	if err != nil {
+		return nil, err
+	}
+	defer r.Close()
+
+	if limit <= 0 {
+		return ioutil.ReadAll(r)
+	}
+
+	size := f.Size
+	if limit > 0 && size > limit {
+		size = limit
+	}
+	buf := bytes.NewBuffer(nil)
+	buf.Grow(int(size))
+	_, err = io.Copy(buf, io.LimitReader(r, limit))
+	return buf.Bytes(), err
+}

modules/git/repo_language_stats_nogogit.go

@@ -0,0 +1,109 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build !gogit
+
+package git
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+
+	"code.gitea.io/gitea/modules/analyze"
+
+	"github.com/go-enry/go-enry/v2"
+)
+
+// GetLanguageStats calculates language stats for git repository at specified commit
+func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, error) {
+	// FIXME: We can be more efficient here...
+	//
+	// We're expecting that we will be reading a lot of blobs and the trees
+	// Thus we should use a shared `cat-file --batch` to get all of this data
+	// And keep the buffers around with resets as necessary.
+	//
+	// It's more complicated so...
+	commit, err := repo.GetCommit(commitID)
+	if err != nil {
+		log("Unable to get commit for: %s", commitID)
+		return nil, err
+	}
+
+	tree := commit.Tree
+
+	entries, err := tree.ListEntriesRecursive()
+	if err != nil {
+		return nil, err
+	}
+
+	sizes := make(map[string]int64)
+	for _, f := range entries {
+		if f.Size() == 0 || enry.IsVendor(f.Name()) || enry.IsDotFile(f.Name()) ||
+			enry.IsDocumentation(f.Name()) || enry.IsConfiguration(f.Name()) {
+			continue
+		}
+
+		// If content can not be read or file is too big just do detection by filename
+		var content []byte
+		if f.Size() <= bigFileSize {
+			content, _ = readFile(f, fileSizeLimit)
+		}
+		if enry.IsGenerated(f.Name(), content) {
+			continue
+		}
+
+		// TODO: Use .gitattributes file for linguist overrides
+		// FIXME: Why can't we split this and the IsGenerated tests to avoid reading the blob unless absolutely necessary?
+		// - eg. do the all the detection tests using filename first before reading content.
+		language := analyze.GetCodeLanguage(f.Name(), content)
+		if language == enry.OtherLanguage || language == "" {
+			continue
+		}
+
+		// group languages, such as Pug -> HTML; SCSS -> CSS
+		group := enry.GetLanguageGroup(language)
+		if group != "" {
+			language = group
+		}
+
+		sizes[language] += f.Size()
+
+		continue
+	}
+
+	// filter special languages unless they are the only language
+	if len(sizes) > 1 {
+		for language := range sizes {
+			langtype := enry.GetLanguageType(language)
+			if langtype != enry.Programming && langtype != enry.Markup {
+				delete(sizes, language)
+			}
+		}
+	}
+
+	return sizes, nil
+}
+
+func readFile(entry *TreeEntry, limit int64) ([]byte, error) {
+	// FIXME: We can probably be a little more efficient here... see above
+	r, err := entry.Blob().DataAsync()
+	if err != nil {
+		return nil, err
+	}
+	defer r.Close()
+
+	if limit <= 0 {
+		return ioutil.ReadAll(r)
+	}
+
+	size := entry.Size()
+	if limit > 0 && size > limit {
+		size = limit
+	}
+	buf := bytes.NewBuffer(nil)
+	buf.Grow(int(size))
+	_, err = io.Copy(buf, io.LimitReader(r, limit))
+	return buf.Bytes(), err
+}

modules/git/repo_object.go

@@ -27,6 +27,11 @@ const (
 	ObjectBranch ObjectType = "branch"
 )
 
+// Bytes returns the byte array for the Object Type
+func (o ObjectType) Bytes() []byte {
+	return []byte(o)
+}
+
 // HashObject takes a reader and returns SHA1 hash for that reader
 func (repo *Repository) HashObject(reader io.Reader) (SHA1, error) {
 	idStr, err := repo.hashObject(reader)

modules/git/repo_ref.go

@@ -4,52 +4,7 @@
 
 package git
 
-import (
-	"strings"
-
-	"github.com/go-git/go-git/v5"
-	"github.com/go-git/go-git/v5/plumbing"
-)
-
 // GetRefs returns all references of the repository.
 func (repo *Repository) GetRefs() ([]*Reference, error) {
 	return repo.GetRefsFiltered("")
 }
-
-// GetRefsFiltered returns all references of the repository that matches patterm exactly or starting with.
-func (repo *Repository) GetRefsFiltered(pattern string) ([]*Reference, error) {
-	r, err := git.PlainOpen(repo.Path)
-	if err != nil {
-		return nil, err
-	}
-
-	refsIter, err := r.References()
-	if err != nil {
-		return nil, err
-	}
-	refs := make([]*Reference, 0)
-	if err = refsIter.ForEach(func(ref *plumbing.Reference) error {
-		if ref.Name() != plumbing.HEAD && !ref.Name().IsRemote() &&
-			(pattern == "" || strings.HasPrefix(ref.Name().String(), pattern)) {
-			refType := string(ObjectCommit)
-			if ref.Name().IsTag() {
-				// tags can be of type `commit` (lightweight) or `tag` (annotated)
-				if tagType, _ := repo.GetTagType(ref.Hash()); err == nil {
-					refType = tagType
-				}
-			}
-			r := &Reference{
-				Name:   ref.Name().String(),
-				Object: ref.Hash(),
-				Type:   refType,
-				repo:   repo,
-			}
-			refs = append(refs, r)
-		}
-		return nil
-	}); err != nil {
-		return nil, err
-	}
-
-	return refs, nil
-}

modules/git/repo_ref_gogit.go

@@ -0,0 +1,52 @@
+// Copyright 2018 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build gogit
+
+package git
+
+import (
+	"strings"
+
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing"
+)
+
+// GetRefsFiltered returns all references of the repository that matches patterm exactly or starting with.
+func (repo *Repository) GetRefsFiltered(pattern string) ([]*Reference, error) {
+	r, err := git.PlainOpen(repo.Path)
+	if err != nil {
+		return nil, err
+	}
+
+	refsIter, err := r.References()
+	if err != nil {
+		return nil, err
+	}
+	refs := make([]*Reference, 0)
+	if err = refsIter.ForEach(func(ref *plumbing.Reference) error {
+		if ref.Name() != plumbing.HEAD && !ref.Name().IsRemote() &&
+			(pattern == "" || strings.HasPrefix(ref.Name().String(), pattern)) {
+			refType := string(ObjectCommit)
+			if ref.Name().IsTag() {
+				// tags can be of type `commit` (lightweight) or `tag` (annotated)
+				if tagType, _ := repo.GetTagType(ref.Hash()); err == nil {
+					refType = tagType
+				}
+			}
+			r := &Reference{
+				Name:   ref.Name().String(),
+				Object: ref.Hash(),
+				Type:   refType,
+				repo:   repo,
+			}
+			refs = append(refs, r)
+		}
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	return refs, nil
+}

modules/git/repo_ref_nogogit.go

@@ -0,0 +1,84 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build !gogit
+
+package git
+
+import (
+	"bufio"
+	"io"
+	"strings"
+)
+
+// GetRefsFiltered returns all references of the repository that matches patterm exactly or starting with.
+func (repo *Repository) GetRefsFiltered(pattern string) ([]*Reference, error) {
+	stdoutReader, stdoutWriter := io.Pipe()
+	defer func() {
+		_ = stdoutReader.Close()
+		_ = stdoutWriter.Close()
+	}()
+
+	go func() {
+		stderrBuilder := &strings.Builder{}
+		err := NewCommand("for-each-ref").RunInDirPipeline(repo.Path, stdoutWriter, stderrBuilder)
+		if err != nil {
+			_ = stdoutWriter.CloseWithError(ConcatenateError(err, stderrBuilder.String()))
+		} else {
+			_ = stdoutWriter.Close()
+		}
+	}()
+
+	refs := make([]*Reference, 0)
+	bufReader := bufio.NewReader(stdoutReader)
+	for {
+		// The output of for-each-ref is simply a list:
+		// <sha> SP <type> TAB <ref> LF
+		sha, err := bufReader.ReadString(' ')
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+		sha = sha[:len(sha)-1]
+
+		typ, err := bufReader.ReadString('\t')
+		if err == io.EOF {
+			// This should not happen, but we'll tolerate it
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+		typ = typ[:len(typ)-1]
+
+		refName, err := bufReader.ReadString('\n')
+		if err == io.EOF {
+			// This should not happen, but we'll tolerate it
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+		refName = refName[:len(refName)-1]
+
+		// refName cannot be HEAD but can be remotes or stash
+		if strings.HasPrefix(refName, "/refs/remotes/") || refName == "/refs/stash" {
+			continue
+		}
+
+		if pattern == "" || strings.HasPrefix(refName, pattern) {
+			r := &Reference{
+				Name:   refName,
+				Object: MustIDFromString(sha),
+				Type:   typ,
+				repo:   repo,
+			}
+			refs = append(refs, r)
+		}
+	}
+
+	return refs, nil
+}

modules/git/repo_tag.go

@@ -8,8 +8,6 @@ package git
 import (
 	"fmt"
 	"strings"
-
-	"github.com/go-git/go-git/v5/plumbing"
 )
 
 // TagPrefix tags prefix path on the repository
@@ -20,12 +18,6 @@ func IsTagExist(repoPath, name string) bool {
 	return IsReferenceExist(repoPath, TagPrefix+name)
 }
 
-// IsTagExist returns true if given tag exists in the repository.
-func (repo *Repository) IsTagExist(name string) bool {
-	_, err := repo.gogitRepo.Reference(plumbing.ReferenceName(TagPrefix+name), true)
-	return err == nil
-}
-
 // CreateTag create one tag in the repository
 func (repo *Repository) CreateTag(name, revision string) error {
 	_, err := NewCommand("tag", "--", name, revision).RunInDir(repo.Path)
@@ -224,29 +216,6 @@ func (repo *Repository) GetTagInfos(page, pageSize int) ([]*Tag, error) {
 	return tags, nil
 }
 
-// GetTags returns all tags of the repository.
-func (repo *Repository) GetTags() ([]string, error) {
-	var tagNames []string
-
-	tags, err := repo.gogitRepo.Tags()
-	if err != nil {
-		return nil, err
-	}
-
-	_ = tags.ForEach(func(tag *plumbing.Reference) error {
-		tagNames = append(tagNames, strings.TrimPrefix(tag.Name().String(), TagPrefix))
-		return nil
-	})
-
-	// Reverse order
-	for i := 0; i < len(tagNames)/2; i++ {
-		j := len(tagNames) - i - 1
-		tagNames[i], tagNames[j] = tagNames[j], tagNames[i]
-	}
-
-	return tagNames, nil
-}
-
 // GetTagType gets the type of the tag, either commit (simple) or tag (annotated)
 func (repo *Repository) GetTagType(id SHA1) (string, error) {
 	// Get tag type

modules/git/repo_tag_gogit.go

@@ -0,0 +1,43 @@
+// Copyright 2015 The Gogs Authors. All rights reserved.
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build gogit
+
+package git
+
+import (
+	"strings"
+
+	"github.com/go-git/go-git/v5/plumbing"
+)
+
+// IsTagExist returns true if given tag exists in the repository.
+func (repo *Repository) IsTagExist(name string) bool {
+	_, err := repo.gogitRepo.Reference(plumbing.ReferenceName(TagPrefix+name), true)
+	return err == nil
+}
+
+// GetTags returns all tags of the repository.
+func (repo *Repository) GetTags() ([]string, error) {
+	var tagNames []string
+
+	tags, err := repo.gogitRepo.Tags()
+	if err != nil {
+		return nil, err
+	}
+
+	_ = tags.ForEach(func(tag *plumbing.Reference) error {
+		tagNames = append(tagNames, strings.TrimPrefix(tag.Name().String(), TagPrefix))
+		return nil
+	})
+
+	// Reverse order
+	for i := 0; i < len(tagNames)/2; i++ {
+		j := len(tagNames) - i - 1
+		tagNames[i], tagNames[j] = tagNames[j], tagNames[i]
+	}
+
+	return tagNames, nil
+}

modules/git/repo_tag_nogogit.go

@@ -0,0 +1,18 @@
+// Copyright 2015 The Gogs Authors. All rights reserved.
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build !gogit
+
+package git
+
+// IsTagExist returns true if given tag exists in the repository.
+func (repo *Repository) IsTagExist(name string) bool {
+	return IsReferenceExist(repo.Path, TagPrefix+name)
+}
+
+// GetTags returns all tags of the repository.
+func (repo *Repository) GetTags() ([]string, error) {
+	return callShowRef(repo.Path, TagPrefix, "--tags")
+}

modules/git/repo_tree.go

@@ -13,45 +13,6 @@ import (
 	"time"
 )
 
-func (repo *Repository) getTree(id SHA1) (*Tree, error) {
-	gogitTree, err := repo.gogitRepo.TreeObject(id)
-	if err != nil {
-		return nil, err
-	}
-
-	tree := NewTree(repo, id)
-	tree.gogitTree = gogitTree
-	return tree, nil
-}
-
-// GetTree find the tree object in the repository.
-func (repo *Repository) GetTree(idStr string) (*Tree, error) {
-	if len(idStr) != 40 {
-		res, err := NewCommand("rev-parse", "--verify", idStr).RunInDir(repo.Path)
-		if err != nil {
-			return nil, err
-		}
-		if len(res) > 0 {
-			idStr = res[:len(res)-1]
-		}
-	}
-	id, err := NewIDFromString(idStr)
-	if err != nil {
-		return nil, err
-	}
-	resolvedID := id
-	commitObject, err := repo.gogitRepo.CommitObject(id)
-	if err == nil {
-		id = SHA1(commitObject.TreeHash)
-	}
-	treeObject, err := repo.getTree(id)
-	if err != nil {
-		return nil, err
-	}
-	treeObject.ResolvedID = resolvedID
-	return treeObject, nil
-}
-
 // CommitTreeOpts represents the possible options to CommitTree
 type CommitTreeOpts struct {
 	Parents    []string
@@ -102,7 +63,7 @@ func (repo *Repository) CommitTree(author *Signature, committer *Signature, tree
 	err = cmd.RunInDirTimeoutEnvFullPipeline(env, -1, repo.Path, stdout, stderr, messageBytes)
 
 	if err != nil {
-		return SHA1{}, concatenateError(err, stderr.String())
+		return SHA1{}, ConcatenateError(err, stderr.String())
 	}
 	return NewIDFromString(strings.TrimSpace(stdout.String()))
 }

modules/git/repo_tree_gogit.go

@@ -0,0 +1,47 @@
+// Copyright 2015 The Gogs Authors. All rights reserved.
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build gogit
+
+package git
+
+func (repo *Repository) getTree(id SHA1) (*Tree, error) {
+	gogitTree, err := repo.gogitRepo.TreeObject(id)
+	if err != nil {
+		return nil, err
+	}
+
+	tree := NewTree(repo, id)
+	tree.gogitTree = gogitTree
+	return tree, nil
+}
+
+// GetTree find the tree object in the repository.
+func (repo *Repository) GetTree(idStr string) (*Tree, error) {
+	if len(idStr) != 40 {
+		res, err := NewCommand("rev-parse", "--verify", idStr).RunInDir(repo.Path)
+		if err != nil {
+			return nil, err
+		}
+		if len(res) > 0 {
+			idStr = res[:len(res)-1]
+		}
+	}
+	id, err := NewIDFromString(idStr)
+	if err != nil {
+		return nil, err
+	}
+	resolvedID := id
+	commitObject, err := repo.gogitRepo.CommitObject(id)
+	if err == nil {
+		id = SHA1(commitObject.TreeHash)
+	}
+	treeObject, err := repo.getTree(id)
+	if err != nil {
+		return nil, err
+	}
+	treeObject.ResolvedID = resolvedID
+	return treeObject, nil
+}

modules/git/repo_tree_nogogit.go

@@ -0,0 +1,98 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build !gogit
+
+package git
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"strings"
+)
+
+func (repo *Repository) getTree(id SHA1) (*Tree, error) {
+	stdoutReader, stdoutWriter := io.Pipe()
+	defer func() {
+		_ = stdoutReader.Close()
+		_ = stdoutWriter.Close()
+	}()
+
+	go func() {
+		stderr := &strings.Builder{}
+		err := NewCommand("cat-file", "--batch").RunInDirFullPipeline(repo.Path, stdoutWriter, stderr, strings.NewReader(id.String()+"\n"))
+		if err != nil {
+			_ = stdoutWriter.CloseWithError(ConcatenateError(err, stderr.String()))
+		} else {
+			_ = stdoutWriter.Close()
+		}
+	}()
+
+	bufReader := bufio.NewReader(stdoutReader)
+	// ignore the SHA
+	_, typ, _, err := ReadBatchLine(bufReader)
+	if err != nil {
+		return nil, err
+	}
+
+	switch typ {
+	case "tag":
+		resolvedID := id
+		data, err := ioutil.ReadAll(bufReader)
+		if err != nil {
+			return nil, err
+		}
+		tag, err := parseTagData(data)
+		if err != nil {
+			return nil, err
+		}
+		commit, err := tag.Commit()
+		if err != nil {
+			return nil, err
+		}
+		commit.Tree.ResolvedID = resolvedID
+		log("tag.commit.Tree: %s %v", commit.Tree.ID.String(), commit.Tree.repo)
+		return &commit.Tree, nil
+	case "commit":
+		commit, err := CommitFromReader(repo, id, bufReader)
+		if err != nil {
+			_ = stdoutReader.CloseWithError(err)
+			return nil, err
+		}
+		commit.Tree.ResolvedID = commit.ID
+		log("commit.Tree: %s %v", commit.Tree.ID.String(), commit.Tree.repo)
+		return &commit.Tree, nil
+	case "tree":
+		stdoutReader.Close()
+		tree := NewTree(repo, id)
+		tree.ResolvedID = id
+		return tree, nil
+	default:
+		_ = stdoutReader.CloseWithError(fmt.Errorf("unknown typ: %s", typ))
+		return nil, ErrNotExist{
+			ID: id.String(),
+		}
+	}
+}
+
+// GetTree find the tree object in the repository.
+func (repo *Repository) GetTree(idStr string) (*Tree, error) {
+	if len(idStr) != 40 {
+		res, err := NewCommand("rev-parse", "--verify", idStr).RunInDir(repo.Path)
+		if err != nil {
+			return nil, err
+		}
+		if len(res) > 0 {
+			idStr = res[:len(res)-1]
+		}
+	}
+	id, err := NewIDFromString(idStr)
+	if err != nil {
+		return nil, err
+	}
+
+	return repo.getTree(id)
+}

modules/git/sha1.go

@@ -10,8 +10,6 @@ import (
 	"fmt"
 	"regexp"
 	"strings"
-
-	"github.com/go-git/go-git/v5/plumbing"
 )
 
 // EmptySHA defines empty git SHA
@@ -23,9 +21,6 @@ const EmptyTreeSHA = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"
 // SHAPattern can be used to determine if a string is an valid sha
 var SHAPattern = regexp.MustCompile(`^[0-9a-f]{4,40}$`)
 
-// SHA1 a git commit name
-type SHA1 = plumbing.Hash
-
 // MustID always creates a new SHA1 from a [20]byte array with no validation of input.
 func MustID(b []byte) SHA1 {
 	var id SHA1

modules/git/sha1_gogit.go

@@ -0,0 +1,20 @@
+// Copyright 2015 The Gogs Authors. All rights reserved.
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build gogit
+
+package git
+
+import (
+	"github.com/go-git/go-git/v5/plumbing"
+)
+
+// SHA1 a git commit name
+type SHA1 = plumbing.Hash
+
+// ComputeBlobHash compute the hash for a given blob content
+func ComputeBlobHash(content []byte) SHA1 {
+	return plumbing.ComputeHash(plumbing.BlobObject, content)
+}

modules/git/sha1_nogogit.go

@@ -0,0 +1,62 @@
+// Copyright 2015 The Gogs Authors. All rights reserved.
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build !gogit
+
+package git
+
+import (
+	"crypto/sha1"
+	"encoding/hex"
+	"hash"
+	"strconv"
+)
+
+// SHA1 a git commit name
+type SHA1 [20]byte
+
+// String returns a string representation of the SHA
+func (s SHA1) String() string {
+	return hex.EncodeToString(s[:])
+}
+
+// IsZero returns whether this SHA1 is all zeroes
+func (s SHA1) IsZero() bool {
+	var empty SHA1
+	return s == empty
+}
+
+// ComputeBlobHash compute the hash for a given blob content
+func ComputeBlobHash(content []byte) SHA1 {
+	return ComputeHash(ObjectBlob, content)
+}
+
+// ComputeHash compute the hash for a given ObjectType and content
+func ComputeHash(t ObjectType, content []byte) SHA1 {
+	h := NewHasher(t, int64(len(content)))
+	_, _ = h.Write(content)
+	return h.Sum()
+}
+
+// Hasher is a struct that will generate a SHA1
+type Hasher struct {
+	hash.Hash
+}
+
+// NewHasher takes an object type and size and creates a hasher to generate a SHA
+func NewHasher(t ObjectType, size int64) Hasher {
+	h := Hasher{sha1.New()}
+	_, _ = h.Write(t.Bytes())
+	_, _ = h.Write([]byte(" "))
+	_, _ = h.Write([]byte(strconv.FormatInt(size, 10)))
+	_, _ = h.Write([]byte{0})
+	return h
+}
+
+// Sum generates a SHA1 for the provided hash
+func (h Hasher) Sum() (sha1 SHA1) {
+	copy(sha1[:], h.Hash.Sum(nil))
+	return
+}

modules/git/signature.go

@@ -5,53 +5,7 @@
 
 package git
 
-import (
-	"bytes"
-	"strconv"
-	"time"
-
-	"github.com/go-git/go-git/v5/plumbing/object"
-)
-
-// Signature represents the Author or Committer information.
-type Signature = object.Signature
-
 const (
 	// GitTimeLayout is the (default) time layout used by git.
 	GitTimeLayout = "Mon Jan _2 15:04:05 2006 -0700"
 )
-
-// Helper to get a signature from the commit line, which looks like these:
-//     author Patrick Gundlach <gundlach@speedata.de> 1378823654 +0200
-//     author Patrick Gundlach <gundlach@speedata.de> Thu, 07 Apr 2005 22:13:13 +0200
-// but without the "author " at the beginning (this method should)
-// be used for author and committer.
-//
-// FIXME: include timezone for timestamp!
-func newSignatureFromCommitline(line []byte) (_ *Signature, err error) {
-	sig := new(Signature)
-	emailStart := bytes.IndexByte(line, '<')
-	sig.Name = string(line[:emailStart-1])
-	emailEnd := bytes.IndexByte(line, '>')
-	sig.Email = string(line[emailStart+1 : emailEnd])
-
-	// Check date format.
-	if len(line) > emailEnd+2 {
-		firstChar := line[emailEnd+2]
-		if firstChar >= 48 && firstChar <= 57 {
-			timestop := bytes.IndexByte(line[emailEnd+2:], ' ')
-			timestring := string(line[emailEnd+2 : emailEnd+2+timestop])
-			seconds, _ := strconv.ParseInt(timestring, 10, 64)
-			sig.When = time.Unix(seconds, 0)
-		} else {
-			sig.When, err = time.Parse(GitTimeLayout, string(line[emailEnd+2:]))
-			if err != nil {
-				return nil, err
-			}
-		}
-	} else {
-		// Fall back to unix 0 time
-		sig.When = time.Unix(0, 0)
-	}
-	return sig, nil
-}

modules/git/signature_gogit.go

@@ -0,0 +1,54 @@
+// Copyright 2015 The Gogs Authors. All rights reserved.
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build gogit
+
+package git
+
+import (
+	"bytes"
+	"strconv"
+	"time"
+
+	"github.com/go-git/go-git/v5/plumbing/object"
+)
+
+// Signature represents the Author or Committer information.
+type Signature = object.Signature
+
+// Helper to get a signature from the commit line, which looks like these:
+//     author Patrick Gundlach <gundlach@speedata.de> 1378823654 +0200
+//     author Patrick Gundlach <gundlach@speedata.de> Thu, 07 Apr 2005 22:13:13 +0200
+// but without the "author " at the beginning (this method should)
+// be used for author and committer.
+//
+// FIXME: include timezone for timestamp!
+func newSignatureFromCommitline(line []byte) (_ *Signature, err error) {
+	sig := new(Signature)
+	emailStart := bytes.IndexByte(line, '<')
+	sig.Name = string(line[:emailStart-1])
+	emailEnd := bytes.IndexByte(line, '>')
+	sig.Email = string(line[emailStart+1 : emailEnd])
+
+	// Check date format.
+	if len(line) > emailEnd+2 {
+		firstChar := line[emailEnd+2]
+		if firstChar >= 48 && firstChar <= 57 {
+			timestop := bytes.IndexByte(line[emailEnd+2:], ' ')
+			timestring := string(line[emailEnd+2 : emailEnd+2+timestop])
+			seconds, _ := strconv.ParseInt(timestring, 10, 64)
+			sig.When = time.Unix(seconds, 0)
+		} else {
+			sig.When, err = time.Parse(GitTimeLayout, string(line[emailEnd+2:]))
+			if err != nil {
+				return nil, err
+			}
+		}
+	} else {
+		// Fall back to unix 0 time
+		sig.When = time.Unix(0, 0)
+	}
+	return sig, nil
+}

modules/git/signature_nogogit.go

@@ -0,0 +1,95 @@
+// Copyright 2015 The Gogs Authors. All rights reserved.
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build !gogit
+
+package git
+
+import (
+	"bytes"
+	"fmt"
+	"strconv"
+	"time"
+)
+
+// Signature represents the Author or Committer information.
+type Signature struct {
+	// Name represents a person name. It is an arbitrary string.
+	Name string
+	// Email is an email, but it cannot be assumed to be well-formed.
+	Email string
+	// When is the timestamp of the signature.
+	When time.Time
+}
+
+func (s *Signature) String() string {
+	return fmt.Sprintf("%s <%s>", s.Name, s.Email)
+}
+
+// Decode decodes a byte array representing a signature to signature
+func (s *Signature) Decode(b []byte) {
+	sig, _ := newSignatureFromCommitline(b)
+	s.Email = sig.Email
+	s.Name = sig.Name
+	s.When = sig.When
+}
+
+// Helper to get a signature from the commit line, which looks like these:
+//     author Patrick Gundlach <gundlach@speedata.de> 1378823654 +0200
+//     author Patrick Gundlach <gundlach@speedata.de> Thu, 07 Apr 2005 22:13:13 +0200
+// but without the "author " at the beginning (this method should)
+// be used for author and committer.
+func newSignatureFromCommitline(line []byte) (sig *Signature, err error) {
+	sig = new(Signature)
+	emailStart := bytes.LastIndexByte(line, '<')
+	emailEnd := bytes.LastIndexByte(line, '>')
+	if emailStart == -1 || emailEnd == -1 || emailEnd < emailStart {
+		return
+	}
+
+	sig.Name = string(line[:emailStart-1])
+	sig.Email = string(line[emailStart+1 : emailEnd])
+
+	hasTime := emailEnd+2 < len(line)
+	if !hasTime {
+		return
+	}
+
+	// Check date format.
+	firstChar := line[emailEnd+2]
+	if firstChar >= 48 && firstChar <= 57 {
+		idx := bytes.IndexByte(line[emailEnd+2:], ' ')
+		if idx < 0 {
+			return
+		}
+
+		timestring := string(line[emailEnd+2 : emailEnd+2+idx])
+		seconds, _ := strconv.ParseInt(timestring, 10, 64)
+		sig.When = time.Unix(seconds, 0)
+
+		idx += emailEnd + 3
+		if idx >= len(line) || idx+5 > len(line) {
+			return
+		}
+
+		timezone := string(line[idx : idx+5])
+		tzhours, err1 := strconv.ParseInt(timezone[0:3], 10, 64)
+		tzmins, err2 := strconv.ParseInt(timezone[3:], 10, 64)
+		if err1 != nil || err2 != nil {
+			return
+		}
+		if tzhours < 0 {
+			tzmins *= -1
+		}
+		tz := time.FixedZone("", int(tzhours*60*60+tzmins*60))
+		sig.When = sig.When.In(tz)
+	} else {
+		sig.When, err = time.Parse(GitTimeLayout, string(line[emailEnd+2:]))
+		if err != nil {
+			return
+		}
+	}
+	return
+}

modules/git/tag.go

@@ -10,6 +10,9 @@ import (
 	"strings"
 )
 
+const beginpgp = "\n-----BEGIN PGP SIGNATURE-----\n"
+const endpgp = "\n-----END PGP SIGNATURE-----"
+
 // Tag represents a Git tag.
 type Tag struct {
 	Name      string
@@ -19,6 +22,7 @@ type Tag struct {
 	Type      string
 	Tagger    *Signature
 	Message   string
+	Signature *CommitGPGSignature
 }
 
 // Commit return the commit of the tag reference
@@ -60,12 +64,23 @@ l:
 			}
 			nextline += eol + 1
 		case eol == 0:
-			tag.Message = strings.TrimRight(string(data[nextline+1:]), "\n")
+			tag.Message = string(data[nextline+1 : len(data)-1])
 			break l
 		default:
 			break l
 		}
 	}
+	idx := strings.LastIndex(tag.Message, beginpgp)
+	if idx > 0 {
+		endSigIdx := strings.Index(tag.Message[idx:], endpgp)
+		if endSigIdx > 0 {
+			tag.Signature = &CommitGPGSignature{
+				Signature: tag.Message[idx+1 : idx+endSigIdx+len(endpgp)],
+				Payload:   string(data[:bytes.LastIndex(data, []byte(beginpgp))+1]),
+			}
+			tag.Message = tag.Message[:idx+1]
+		}
+	}
 	return tag, nil
 }
 

modules/git/tree.go

@@ -6,25 +6,9 @@
 package git
 
 import (
-	"io"
 	"strings"
-
-	"github.com/go-git/go-git/v5/plumbing"
-	"github.com/go-git/go-git/v5/plumbing/object"
 )
 
-// Tree represents a flat directory listing.
-type Tree struct {
-	ID         SHA1
-	ResolvedID SHA1
-	repo       *Repository
-
-	gogitTree *object.Tree
-
-	// parent tree
-	ptree *Tree
-}
-
 // NewTree create a new tree according the repository and tree id
 func NewTree(repo *Repository, id SHA1) *Tree {
 	return &Tree{
@@ -61,70 +45,3 @@ func (t *Tree) SubTree(rpath string) (*Tree, error) {
 	}
 	return g, nil
 }
-
-func (t *Tree) loadTreeObject() error {
-	gogitTree, err := t.repo.gogitRepo.TreeObject(t.ID)
-	if err != nil {
-		return err
-	}
-
-	t.gogitTree = gogitTree
-	return nil
-}
-
-// ListEntries returns all entries of current tree.
-func (t *Tree) ListEntries() (Entries, error) {
-	if t.gogitTree == nil {
-		err := t.loadTreeObject()
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	entries := make([]*TreeEntry, len(t.gogitTree.Entries))
-	for i, entry := range t.gogitTree.Entries {
-		entries[i] = &TreeEntry{
-			ID:             entry.Hash,
-			gogitTreeEntry: &t.gogitTree.Entries[i],
-			ptree:          t,
-		}
-	}
-
-	return entries, nil
-}
-
-// ListEntriesRecursive returns all entries of current tree recursively including all subtrees
-func (t *Tree) ListEntriesRecursive() (Entries, error) {
-	if t.gogitTree == nil {
-		err := t.loadTreeObject()
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	var entries []*TreeEntry
-	seen := map[plumbing.Hash]bool{}
-	walker := object.NewTreeWalker(t.gogitTree, true, seen)
-	for {
-		fullName, entry, err := walker.Next()
-		if err == io.EOF {
-			break
-		}
-		if err != nil {
-			return nil, err
-		}
-		if seen[entry.Hash] {
-			continue
-		}
-
-		convertedEntry := &TreeEntry{
-			ID:             entry.Hash,
-			gogitTreeEntry: &entry,
-			ptree:          t,
-			fullName:       fullName,
-		}
-		entries = append(entries, convertedEntry)
-	}
-
-	return entries, nil
-}

modules/git/tree_blob.go

@@ -5,64 +5,6 @@
 
 package git
 
-import (
-	"path"
-	"strings"
-
-	"github.com/go-git/go-git/v5/plumbing"
-	"github.com/go-git/go-git/v5/plumbing/filemode"
-	"github.com/go-git/go-git/v5/plumbing/object"
-)
-
-// GetTreeEntryByPath get the tree entries according the sub dir
-func (t *Tree) GetTreeEntryByPath(relpath string) (*TreeEntry, error) {
-	if len(relpath) == 0 {
-		return &TreeEntry{
-			ID: t.ID,
-			//Type: ObjectTree,
-			gogitTreeEntry: &object.TreeEntry{
-				Name: "",
-				Mode: filemode.Dir,
-				Hash: t.ID,
-			},
-		}, nil
-	}
-
-	relpath = path.Clean(relpath)
-	parts := strings.Split(relpath, "/")
-	var err error
-	tree := t
-	for i, name := range parts {
-		if i == len(parts)-1 {
-			entries, err := tree.ListEntries()
-			if err != nil {
-				if err == plumbing.ErrObjectNotFound {
-					return nil, ErrNotExist{
-						RelPath: relpath,
-					}
-				}
-				return nil, err
-			}
-			for _, v := range entries {
-				if v.Name() == name {
-					return v, nil
-				}
-			}
-		} else {
-			tree, err = tree.SubTree(name)
-			if err != nil {
-				if err == plumbing.ErrObjectNotFound {
-					return nil, ErrNotExist{
-						RelPath: relpath,
-					}
-				}
-				return nil, err
-			}
-		}
-	}
-	return nil, ErrNotExist{"", relpath}
-}
-
 // GetBlobByPath get the blob object according the path
 func (t *Tree) GetBlobByPath(relpath string) (*Blob, error) {
 	entry, err := t.GetTreeEntryByPath(relpath)

modules/git/tree_blob_gogit.go

@@ -0,0 +1,66 @@
+// Copyright 2015 The Gogs Authors. All rights reserved.
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build gogit
+
+package git
+
+import (
+	"path"
+	"strings"
+
+	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/go-git/go-git/v5/plumbing/filemode"
+	"github.com/go-git/go-git/v5/plumbing/object"
+)
+
+// GetTreeEntryByPath get the tree entries according the sub dir
+func (t *Tree) GetTreeEntryByPath(relpath string) (*TreeEntry, error) {
+	if len(relpath) == 0 {
+		return &TreeEntry{
+			ID: t.ID,
+			//Type: ObjectTree,
+			gogitTreeEntry: &object.TreeEntry{
+				Name: "",
+				Mode: filemode.Dir,
+				Hash: t.ID,
+			},
+		}, nil
+	}
+
+	relpath = path.Clean(relpath)
+	parts := strings.Split(relpath, "/")
+	var err error
+	tree := t
+	for i, name := range parts {
+		if i == len(parts)-1 {
+			entries, err := tree.ListEntries()
+			if err != nil {
+				if err == plumbing.ErrObjectNotFound {
+					return nil, ErrNotExist{
+						RelPath: relpath,
+					}
+				}
+				return nil, err
+			}
+			for _, v := range entries {
+				if v.Name() == name {
+					return v, nil
+				}
+			}
+		} else {
+			tree, err = tree.SubTree(name)
+			if err != nil {
+				if err == plumbing.ErrObjectNotFound {
+					return nil, ErrNotExist{
+						RelPath: relpath,
+					}
+				}
+				return nil, err
+			}
+		}
+	}
+	return nil, ErrNotExist{"", relpath}
+}

modules/git/tree_blob_nogogit.go

@@ -0,0 +1,49 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build !gogit
+
+package git
+
+import (
+	"path"
+	"strings"
+)
+
+// GetTreeEntryByPath get the tree entries according the sub dir
+func (t *Tree) GetTreeEntryByPath(relpath string) (*TreeEntry, error) {
+	if len(relpath) == 0 {
+		return &TreeEntry{
+			ID:        t.ID,
+			name:      "",
+			fullName:  "",
+			entryMode: EntryModeTree,
+		}, nil
+	}
+
+	// FIXME: This should probably use git cat-file --batch to be a bit more efficient
+	relpath = path.Clean(relpath)
+	parts := strings.Split(relpath, "/")
+	var err error
+	tree := t
+	for i, name := range parts {
+		if i == len(parts)-1 {
+			entries, err := tree.ListEntries()
+			if err != nil {
+				return nil, err
+			}
+			for _, v := range entries {
+				if v.Name() == name {
+					return v, nil
+				}
+			}
+		} else {
+			tree, err = tree.SubTree(name)
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+	return nil, ErrNotExist{"", relpath}
+}

modules/git/tree_entry.go

@@ -9,55 +9,8 @@ import (
 	"io"
 	"sort"
 	"strings"
-
-	"github.com/go-git/go-git/v5/plumbing"
-	"github.com/go-git/go-git/v5/plumbing/filemode"
-	"github.com/go-git/go-git/v5/plumbing/object"
 )
 
-// EntryMode the type of the object in the git tree
-type EntryMode int
-
-// There are only a few file modes in Git. They look like unix file modes, but they can only be
-// one of these.
-const (
-	// EntryModeBlob
-	EntryModeBlob EntryMode = 0100644
-	// EntryModeExec
-	EntryModeExec EntryMode = 0100755
-	// EntryModeSymlink
-	EntryModeSymlink EntryMode = 0120000
-	// EntryModeCommit
-	EntryModeCommit EntryMode = 0160000
-	// EntryModeTree
-	EntryModeTree EntryMode = 0040000
-)
-
-// TreeEntry the leaf in the git tree
-type TreeEntry struct {
-	ID SHA1
-
-	gogitTreeEntry *object.TreeEntry
-	ptree          *Tree
-
-	size     int64
-	sized    bool
-	fullName string
-}
-
-// Name returns the name of the entry
-func (te *TreeEntry) Name() string {
-	if te.fullName != "" {
-		return te.fullName
-	}
-	return te.gogitTreeEntry.Name
-}
-
-// Mode returns the mode of the entry
-func (te *TreeEntry) Mode() EntryMode {
-	return EntryMode(te.gogitTreeEntry.Mode)
-}
-
 // Type returns the type of the entry (commit, tree, blob)
 func (te *TreeEntry) Type() string {
 	switch te.Mode() {
@@ -70,63 +23,6 @@ func (te *TreeEntry) Type() string {
 	}
 }
 
-// Size returns the size of the entry
-func (te *TreeEntry) Size() int64 {
-	if te.IsDir() {
-		return 0
-	} else if te.sized {
-		return te.size
-	}
-
-	file, err := te.ptree.gogitTree.TreeEntryFile(te.gogitTreeEntry)
-	if err != nil {
-		return 0
-	}
-
-	te.sized = true
-	te.size = file.Size
-	return te.size
-}
-
-// IsSubModule if the entry is a sub module
-func (te *TreeEntry) IsSubModule() bool {
-	return te.gogitTreeEntry.Mode == filemode.Submodule
-}
-
-// IsDir if the entry is a sub dir
-func (te *TreeEntry) IsDir() bool {
-	return te.gogitTreeEntry.Mode == filemode.Dir
-}
-
-// IsLink if the entry is a symlink
-func (te *TreeEntry) IsLink() bool {
-	return te.gogitTreeEntry.Mode == filemode.Symlink
-}
-
-// IsRegular if the entry is a regular file
-func (te *TreeEntry) IsRegular() bool {
-	return te.gogitTreeEntry.Mode == filemode.Regular
-}
-
-// IsExecutable if the entry is an executable file (not necessarily binary)
-func (te *TreeEntry) IsExecutable() bool {
-	return te.gogitTreeEntry.Mode == filemode.Executable
-}
-
-// Blob returns the blob object the entry
-func (te *TreeEntry) Blob() *Blob {
-	encodedObj, err := te.ptree.repo.gogitRepo.Storer.EncodedObject(plumbing.AnyObject, te.gogitTreeEntry.Hash)
-	if err != nil {
-		return nil
-	}
-
-	return &Blob{
-		ID:              te.gogitTreeEntry.Hash,
-		gogitEncodedObj: encodedObj,
-		name:            te.Name(),
-	}
-}
-
 // FollowLink returns the entry pointed to by a symlink
 func (te *TreeEntry) FollowLink() (*TreeEntry, error) {
 	if !te.IsLink() {

modules/git/tree_entry_gogit.go

@@ -0,0 +1,96 @@
+// Copyright 2015 The Gogs Authors. All rights reserved.
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build gogit
+
+package git
+
+import (
+	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/go-git/go-git/v5/plumbing/filemode"
+	"github.com/go-git/go-git/v5/plumbing/object"
+)
+
+// TreeEntry the leaf in the git tree
+type TreeEntry struct {
+	ID SHA1
+
+	gogitTreeEntry *object.TreeEntry
+	ptree          *Tree
+
+	size     int64
+	sized    bool
+	fullName string
+}
+
+// Name returns the name of the entry
+func (te *TreeEntry) Name() string {
+	if te.fullName != "" {
+		return te.fullName
+	}
+	return te.gogitTreeEntry.Name
+}
+
+// Mode returns the mode of the entry
+func (te *TreeEntry) Mode() EntryMode {
+	return EntryMode(te.gogitTreeEntry.Mode)
+}
+
+// Size returns the size of the entry
+func (te *TreeEntry) Size() int64 {
+	if te.IsDir() {
+		return 0
+	} else if te.sized {
+		return te.size
+	}
+
+	file, err := te.ptree.gogitTree.TreeEntryFile(te.gogitTreeEntry)
+	if err != nil {
+		return 0
+	}
+
+	te.sized = true
+	te.size = file.Size
+	return te.size
+}
+
+// IsSubModule if the entry is a sub module
+func (te *TreeEntry) IsSubModule() bool {
+	return te.gogitTreeEntry.Mode == filemode.Submodule
+}
+
+// IsDir if the entry is a sub dir
+func (te *TreeEntry) IsDir() bool {
+	return te.gogitTreeEntry.Mode == filemode.Dir
+}
+
+// IsLink if the entry is a symlink
+func (te *TreeEntry) IsLink() bool {
+	return te.gogitTreeEntry.Mode == filemode.Symlink
+}
+
+// IsRegular if the entry is a regular file
+func (te *TreeEntry) IsRegular() bool {
+	return te.gogitTreeEntry.Mode == filemode.Regular
+}
+
+// IsExecutable if the entry is an executable file (not necessarily binary)
+func (te *TreeEntry) IsExecutable() bool {
+	return te.gogitTreeEntry.Mode == filemode.Executable
+}
+
+// Blob returns the blob object the entry
+func (te *TreeEntry) Blob() *Blob {
+	encodedObj, err := te.ptree.repo.gogitRepo.Storer.EncodedObject(plumbing.AnyObject, te.gogitTreeEntry.Hash)
+	if err != nil {
+		return nil
+	}
+
+	return &Blob{
+		ID:              te.gogitTreeEntry.Hash,
+		gogitEncodedObj: encodedObj,
+		name:            te.Name(),
+	}
+}

modules/git/tree_entry_mode.go

@@ -0,0 +1,36 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package git
+
+import "strconv"
+
+// EntryMode the type of the object in the git tree
+type EntryMode int
+
+// There are only a few file modes in Git. They look like unix file modes, but they can only be
+// one of these.
+const (
+	// EntryModeBlob
+	EntryModeBlob EntryMode = 0100644
+	// EntryModeExec
+	EntryModeExec EntryMode = 0100755
+	// EntryModeSymlink
+	EntryModeSymlink EntryMode = 0120000
+	// EntryModeCommit
+	EntryModeCommit EntryMode = 0160000
+	// EntryModeTree
+	EntryModeTree EntryMode = 0040000
+)
+
+// String converts an EntryMode to a string
+func (e EntryMode) String() string {
+	return strconv.FormatInt(int64(e), 8)
+}
+
+// ToEntryMode converts a string to an EntryMode
+func ToEntryMode(value string) EntryMode {
+	v, _ := strconv.ParseInt(value, 8, 32)
+	return EntryMode(v)
+}

modules/git/tree_entry_nogogit.go

@@ -0,0 +1,91 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build !gogit
+
+package git
+
+import (
+	"strconv"
+	"strings"
+)
+
+// TreeEntry the leaf in the git tree
+type TreeEntry struct {
+	ID SHA1
+
+	ptree *Tree
+
+	entryMode EntryMode
+	name      string
+
+	size     int64
+	sized    bool
+	fullName string
+}
+
+// Name returns the name of the entry
+func (te *TreeEntry) Name() string {
+	if te.fullName != "" {
+		return te.fullName
+	}
+	return te.name
+}
+
+// Mode returns the mode of the entry
+func (te *TreeEntry) Mode() EntryMode {
+	return te.entryMode
+}
+
+// Size returns the size of the entry
+func (te *TreeEntry) Size() int64 {
+	if te.IsDir() {
+		return 0
+	} else if te.sized {
+		return te.size
+	}
+
+	stdout, err := NewCommand("cat-file", "-s", te.ID.String()).RunInDir(te.ptree.repo.Path)
+	if err != nil {
+		return 0
+	}
+
+	te.sized = true
+	te.size, _ = strconv.ParseInt(strings.TrimSpace(stdout), 10, 64)
+	return te.size
+}
+
+// IsSubModule if the entry is a sub module
+func (te *TreeEntry) IsSubModule() bool {
+	return te.entryMode == EntryModeCommit
+}
+
+// IsDir if the entry is a sub dir
+func (te *TreeEntry) IsDir() bool {
+	return te.entryMode == EntryModeTree
+}
+
+// IsLink if the entry is a symlink
+func (te *TreeEntry) IsLink() bool {
+	return te.entryMode == EntryModeSymlink
+}
+
+// IsRegular if the entry is a regular file
+func (te *TreeEntry) IsRegular() bool {
+	return te.entryMode == EntryModeBlob
+}
+
+// IsExecutable if the entry is an executable file (not necessarily binary)
+func (te *TreeEntry) IsExecutable() bool {
+	return te.entryMode == EntryModeExec
+}
+
+// Blob returns the blob object the entry
+func (te *TreeEntry) Blob() *Blob {
+	return &Blob{
+		ID:       te.ID,
+		repoPath: te.ptree.repo.Path,
+		name:     te.Name(),
+	}
+}

modules/git/tree_entry_test.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+// +build gogit
+
 package git
 
 import (

modules/git/tree_gogit.go

@@ -0,0 +1,94 @@
+// Copyright 2015 The Gogs Authors. All rights reserved.
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build gogit
+
+package git
+
+import (
+	"io"
+
+	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/go-git/go-git/v5/plumbing/object"
+)
+
+// Tree represents a flat directory listing.
+type Tree struct {
+	ID         SHA1
+	ResolvedID SHA1
+	repo       *Repository
+
+	gogitTree *object.Tree
+
+	// parent tree
+	ptree *Tree
+}
+
+func (t *Tree) loadTreeObject() error {
+	gogitTree, err := t.repo.gogitRepo.TreeObject(t.ID)
+	if err != nil {
+		return err
+	}
+
+	t.gogitTree = gogitTree
+	return nil
+}
+
+// ListEntries returns all entries of current tree.
+func (t *Tree) ListEntries() (Entries, error) {
+	if t.gogitTree == nil {
+		err := t.loadTreeObject()
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	entries := make([]*TreeEntry, len(t.gogitTree.Entries))
+	for i, entry := range t.gogitTree.Entries {
+		entries[i] = &TreeEntry{
+			ID:             entry.Hash,
+			gogitTreeEntry: &t.gogitTree.Entries[i],
+			ptree:          t,
+		}
+	}
+
+	return entries, nil
+}
+
+// ListEntriesRecursive returns all entries of current tree recursively including all subtrees
+func (t *Tree) ListEntriesRecursive() (Entries, error) {
+	if t.gogitTree == nil {
+		err := t.loadTreeObject()
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	var entries []*TreeEntry
+	seen := map[plumbing.Hash]bool{}
+	walker := object.NewTreeWalker(t.gogitTree, true, seen)
+	for {
+		fullName, entry, err := walker.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+		if seen[entry.Hash] {
+			continue
+		}
+
+		convertedEntry := &TreeEntry{
+			ID:             entry.Hash,
+			gogitTreeEntry: &entry,
+			ptree:          t,
+			fullName:       fullName,
+		}
+		entries = append(entries, convertedEntry)
+	}
+
+	return entries, nil
+}

modules/git/tree_nogogit.go

@@ -0,0 +1,69 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build !gogit
+
+package git
+
+import (
+	"strings"
+)
+
+// Tree represents a flat directory listing.
+type Tree struct {
+	ID         SHA1
+	ResolvedID SHA1
+	repo       *Repository
+
+	// parent tree
+	ptree *Tree
+
+	entries       Entries
+	entriesParsed bool
+
+	entriesRecursive       Entries
+	entriesRecursiveParsed bool
+}
+
+// ListEntries returns all entries of current tree.
+func (t *Tree) ListEntries() (Entries, error) {
+	if t.entriesParsed {
+		return t.entries, nil
+	}
+
+	stdout, err := NewCommand("ls-tree", t.ID.String()).RunInDirBytes(t.repo.Path)
+	if err != nil {
+		if strings.Contains(err.Error(), "fatal: Not a valid object name") || strings.Contains(err.Error(), "fatal: not a tree object") {
+			return nil, ErrNotExist{
+				ID: t.ID.String(),
+			}
+		}
+		return nil, err
+	}
+
+	t.entries, err = parseTreeEntries(stdout, t)
+	if err == nil {
+		t.entriesParsed = true
+	}
+
+	return t.entries, err
+}
+
+// ListEntriesRecursive returns all entries of current tree recursively including all subtrees
+func (t *Tree) ListEntriesRecursive() (Entries, error) {
+	if t.entriesRecursiveParsed {
+		return t.entriesRecursive, nil
+	}
+	stdout, err := NewCommand("ls-tree", "-t", "-r", t.ID.String()).RunInDirBytes(t.repo.Path)
+	if err != nil {
+		return nil, err
+	}
+
+	t.entriesRecursive, err = parseTreeEntries(stdout, t)
+	if err == nil {
+		t.entriesRecursiveParsed = true
+	}
+
+	return t.entriesRecursive, err
+}

modules/git/utils.go

@@ -6,6 +6,7 @@ package git
 
 import (
 	"fmt"
+	"io"
 	"os"
 	"strconv"
 	"strings"
@@ -68,11 +69,12 @@ func isExist(path string) bool {
 	return err == nil || os.IsExist(err)
 }
 
-func concatenateError(err error, stderr string) error {
+// ConcatenateError concatenats an error with stderr string
+func ConcatenateError(err error, stderr string) error {
 	if len(stderr) == 0 {
 		return err
 	}
-	return fmt.Errorf("%v - %s", err, stderr)
+	return fmt.Errorf("%w - %s", err, stderr)
 }
 
 // RefEndName return the end name of a ref name
@@ -140,3 +142,29 @@ func ParseBool(value string) (result bool, valid bool) {
 	}
 	return intValue != 0, true
 }
+
+// LimitedReaderCloser is a limited reader closer
+type LimitedReaderCloser struct {
+	R io.Reader
+	C io.Closer
+	N int64
+}
+
+// Read implements io.Reader
+func (l *LimitedReaderCloser) Read(p []byte) (n int, err error) {
+	if l.N <= 0 {
+		_ = l.C.Close()
+		return 0, io.EOF
+	}
+	if int64(len(p)) > l.N {
+		p = p[0:l.N]
+	}
+	n, err = l.R.Read(p)
+	l.N -= int64(n)
+	return
+}
+
+// Close implements io.Closer
+func (l *LimitedReaderCloser) Close() error {
+	return l.C.Close()
+}

modules/indexer/stats/db.go

@@ -7,6 +7,7 @@ package stats
 import (
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/log"
 )
 
 // DBIndexer implements Indexer interface to use database's like search
@@ -37,6 +38,7 @@ func (db *DBIndexer) Index(id int64) error {
 	// Get latest commit for default branch
 	commitID, err := gitRepo.GetBranchCommitID(repo.DefaultBranch)
 	if err != nil {
+		log.Error("Unable to get commit ID for defaultbranch %s in %s", repo.DefaultBranch, repo.RepoPath())
 		return err
 	}
 
@@ -48,6 +50,7 @@ func (db *DBIndexer) Index(id int64) error {
 	// Calculate and save language statistics to database
 	stats, err := gitRepo.GetLanguageStats(commitID)
 	if err != nil {
+		log.Error("Unable to get language stats for ID %s for defaultbranch %s in %s. Error: %v", commitID, repo.DefaultBranch, repo.RepoPath(), err)
 		return err
 	}
 	return repo.UpdateLanguageStats(commitID, stats)

modules/repository/cache.go

@@ -5,57 +5,14 @@
 package repository
 
 import (
-	"path"
 	"strings"
 
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/cache"
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/setting"
-
-	cgobject "github.com/go-git/go-git/v5/plumbing/object/commitgraph"
 )
 
-func recusiveCache(gitRepo *git.Repository, c cgobject.CommitNode, tree *git.Tree, treePath string, ca *cache.LastCommitCache, level int) error {
-	if level == 0 {
-		return nil
-	}
-
-	entries, err := tree.ListEntries()
-	if err != nil {
-		return err
-	}
-
-	entryPaths := make([]string, len(entries))
-	entryMap := make(map[string]*git.TreeEntry)
-	for i, entry := range entries {
-		entryPaths[i] = entry.Name()
-		entryMap[entry.Name()] = entry
-	}
-
-	commits, err := git.GetLastCommitForPaths(c, treePath, entryPaths)
-	if err != nil {
-		return err
-	}
-
-	for entry, cm := range commits {
-		if err := ca.Put(c.ID().String(), path.Join(treePath, entry), cm.ID().String()); err != nil {
-			return err
-		}
-		if entryMap[entry].IsDir() {
-			subTree, err := tree.SubTree(entry)
-			if err != nil {
-				return err
-			}
-			if err := recusiveCache(gitRepo, c, subTree, entry, ca, level-1); err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
 func getRefName(fullRefName string) string {
 	if strings.HasPrefix(fullRefName, git.TagPrefix) {
 		return fullRefName[len(git.TagPrefix):]
@@ -84,14 +41,7 @@ func CacheRef(repo *models.Repository, gitRepo *git.Repository, fullRefName stri
 		return nil
 	}
 
-	commitNodeIndex, _ := gitRepo.CommitNodeIndex()
+	commitCache := git.NewLastCommitCache(repo.FullName(), gitRepo, int64(setting.CacheService.LastCommit.TTL.Seconds()), cache.GetCache())
 
-	c, err := commitNodeIndex.Get(commit.ID)
+	return commitCache.CacheCommit(commit)
-	if err != nil {
-		return err
-	}
-
-	ca := cache.NewLastCommitCache(repo.FullName(), gitRepo, int64(setting.CacheService.LastCommit.TTL.Seconds()))
-
-	return recusiveCache(gitRepo, c, &commit.Tree, "", ca, 1)
 }

routers/private/hook.go

@@ -25,7 +25,6 @@ import (
 	repo_service "code.gitea.io/gitea/services/repository"
 
 	"gitea.com/macaron/macaron"
-	"github.com/go-git/go-git/v5/plumbing"
 )
 
 func verifyCommits(oldCommitID, newCommitID string, repo *git.Repository, env []string) error {
@@ -82,7 +81,7 @@ func readAndVerifyCommit(sha string, repo *git.Repository, env []string) error {
 		_ = stdoutReader.Close()
 		_ = stdoutWriter.Close()
 	}()
-	hash := plumbing.NewHash(sha)
+	hash := git.MustIDFromString(sha)
 
 	return git.NewCommand("cat-file", "commit", sha).
 		RunInDirTimeoutEnvFullPipelineFunc(env, -1, repo.Path,

routers/repo/lfs.go

@@ -12,11 +12,9 @@ import (
 	"io"
 	"io/ioutil"
 	"path"
-	"sort"
 	"strconv"
 	"strings"
 	"sync"
-	"time"
 
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/base"
@@ -29,9 +27,6 @@ import (
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/storage"
 
-	gogit "github.com/go-git/go-git/v5"
-	"github.com/go-git/go-git/v5/plumbing"
-	"github.com/go-git/go-git/v5/plumbing/object"
 	"github.com/unknwon/com"
 )
 
@@ -363,22 +358,6 @@ func LFSDelete(ctx *context.Context) {
 	ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs")
 }
 
-type lfsResult struct {
-	Name           string
-	SHA            string
-	Summary        string
-	When           time.Time
-	ParentHashes   []plumbing.Hash
-	BranchName     string
-	FullCommitName string
-}
-
-type lfsResultSlice []*lfsResult
-
-func (a lfsResultSlice) Len() int           { return len(a) }
-func (a lfsResultSlice) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
-func (a lfsResultSlice) Less(i, j int) bool { return a[j].When.After(a[i].When) }
-
 // LFSFileFind guesses a sha for the provided oid (or uses the provided sha) and then finds the commits that contain this sha
 func LFSFileFind(ctx *context.Context) {
 	if !setting.LFS.StartServer {
@@ -394,140 +373,27 @@ func LFSFileFind(ctx *context.Context) {
 	sha := ctx.Query("sha")
 	ctx.Data["Title"] = oid
 	ctx.Data["PageIsSettingsLFS"] = true
-	var hash plumbing.Hash
+	var hash git.SHA1
 	if len(sha) == 0 {
 		meta := models.LFSMetaObject{Oid: oid, Size: size}
 		pointer := meta.Pointer()
-		hash = plumbing.ComputeHash(plumbing.BlobObject, []byte(pointer))
+		hash = git.ComputeBlobHash([]byte(pointer))
 		sha = hash.String()
 	} else {
-		hash = plumbing.NewHash(sha)
+		hash = git.MustIDFromString(sha)
 	}
 	ctx.Data["LFSFilesLink"] = ctx.Repo.RepoLink + "/settings/lfs"
 	ctx.Data["Oid"] = oid
 	ctx.Data["Size"] = size
 	ctx.Data["SHA"] = sha
 
-	resultsMap := map[string]*lfsResult{}
+	results, err := pipeline.FindLFSFile(ctx.Repo.GitRepo, hash)
-	results := make([]*lfsResult, 0)
-
-	basePath := ctx.Repo.Repository.RepoPath()
-	gogitRepo := ctx.Repo.GitRepo.GoGitRepo()
-
-	commitsIter, err := gogitRepo.Log(&gogit.LogOptions{
-		Order: gogit.LogOrderCommitterTime,
-		All:   true,
-	})
-	if err != nil {
-		log.Error("Failed to get GoGit CommitsIter: %v", err)
-		ctx.ServerError("LFSFind: Iterate Commits", err)
-		return
-	}
-
-	err = commitsIter.ForEach(func(gitCommit *object.Commit) error {
-		tree, err := gitCommit.Tree()
-		if err != nil {
-			return err
-		}
-		treeWalker := object.NewTreeWalker(tree, true, nil)
-		defer treeWalker.Close()
-		for {
-			name, entry, err := treeWalker.Next()
-			if err == io.EOF {
-				break
-			}
-			if entry.Hash == hash {
-				result := lfsResult{
-					Name:         name,
-					SHA:          gitCommit.Hash.String(),
-					Summary:      strings.Split(strings.TrimSpace(gitCommit.Message), "\n")[0],
-					When:         gitCommit.Author.When,
-					ParentHashes: gitCommit.ParentHashes,
-				}
-				resultsMap[gitCommit.Hash.String()+":"+name] = &result
-			}
-		}
-		return nil
-	})
 	if err != nil && err != io.EOF {
-		log.Error("Failure in CommitIter.ForEach: %v", err)
+		log.Error("Failure in FindLFSFile: %v", err)
-		ctx.ServerError("LFSFind: IterateCommits ForEach", err)
+		ctx.ServerError("LFSFind: FindLFSFile.", err)
 		return
 	}
 
-	for _, result := range resultsMap {
-		hasParent := false
-		for _, parentHash := range result.ParentHashes {
-			if _, hasParent = resultsMap[parentHash.String()+":"+result.Name]; hasParent {
-				break
-			}
-		}
-		if !hasParent {
-			results = append(results, result)
-		}
-	}
-
-	sort.Sort(lfsResultSlice(results))
-
-	// Should really use a go-git function here but name-rev is not completed and recapitulating it is not simple
-	shasToNameReader, shasToNameWriter := io.Pipe()
-	nameRevStdinReader, nameRevStdinWriter := io.Pipe()
-	errChan := make(chan error, 1)
-	wg := sync.WaitGroup{}
-	wg.Add(3)
-
-	go func() {
-		defer wg.Done()
-		scanner := bufio.NewScanner(nameRevStdinReader)
-		i := 0
-		for scanner.Scan() {
-			line := scanner.Text()
-			if len(line) == 0 {
-				continue
-			}
-			result := results[i]
-			result.FullCommitName = line
-			result.BranchName = strings.Split(line, "~")[0]
-			i++
-		}
-	}()
-	go pipeline.NameRevStdin(shasToNameReader, nameRevStdinWriter, &wg, basePath)
-	go func() {
-		defer wg.Done()
-		defer shasToNameWriter.Close()
-		for _, result := range results {
-			i := 0
-			if i < len(result.SHA) {
-				n, err := shasToNameWriter.Write([]byte(result.SHA)[i:])
-				if err != nil {
-					errChan <- err
-					break
-				}
-				i += n
-			}
-			n := 0
-			for n < 1 {
-				n, err = shasToNameWriter.Write([]byte{'\n'})
-				if err != nil {
-					errChan <- err
-					break
-				}
-
-			}
-
-		}
-	}()
-
-	wg.Wait()
-
-	select {
-	case err, has := <-errChan:
-		if has {
-			ctx.ServerError("LFSPointerFiles", err)
-		}
-	default:
-	}
-
 	ctx.Data["Results"] = results
 	ctx.HTML(200, tplSettingsLFSFileFind)
 }

routers/repo/view.go

@@ -137,9 +137,9 @@ func renderDirectory(ctx *context.Context, treeLink string) {
 	}
 	entries.CustomSort(base.NaturalSortLess)
 
-	var c git.LastCommitCache
+	var c *git.LastCommitCache
 	if setting.CacheService.LastCommit.Enabled && ctx.Repo.CommitsCount >= setting.CacheService.LastCommit.CommitsCount {
-		c = cache.NewLastCommitCache(ctx.Repo.Repository.FullName(), ctx.Repo.GitRepo, int64(setting.CacheService.LastCommit.TTL.Seconds()))
+		c = git.NewLastCommitCache(ctx.Repo.Repository.FullName(), ctx.Repo.GitRepo, int64(setting.CacheService.LastCommit.TTL.Seconds()), cache.GetCache())
 	}
 
 	var latestCommit *git.Commit

templates/repo/view_list.tmpl

@@ -40,18 +40,19 @@
 			</tr>
 		{{end}}
 		{{range $item := .Files}}
-			{{$entry := index $item 0}}
+			{{$entry := $item.Entry}}
-			{{$commit := index $item 1}}
+			{{$commit := $item.Commit}}
+			{{$subModuleFile := $item.SubModuleFile}}
 			<tr>
 				<td class="name four wide">
 					<span class="truncate">
 						{{if $entry.IsSubModule}}
 							{{svg "octicon-file-submodule"}}
-							{{$refURL := $commit.RefURL AppUrl $.Repository.FullName $.SSHDomain}}
+							{{$refURL := $subModuleFile.RefURL AppUrl $.Repository.FullName $.SSHDomain}}
 							{{if $refURL}}
-								<a href="{{$refURL}}">{{$entry.Name}}</a><span class="at">@</span><a href="{{$refURL}}/commit/{{$commit.RefID}}">{{ShortSha $commit.RefID}}</a>
+								<a href="{{$refURL}}">{{$entry.Name}}</a><span class="at">@</span><a href="{{$refURL}}/commit/{{$subModuleFile.RefID}}">{{ShortSha $subModuleFile.RefID}}</a>
 							{{else}}
-								{{$entry.Name}}<span class="at">@</span>{{ShortSha $commit.RefID}}
+								{{$entry.Name}}<span class="at">@</span>{{ShortSha $subModuleFile.RefID}}
 							{{end}}
 						{{else}}
 							{{if $entry.IsDir}}