From da1edbfb79566e9d31f43ee85eb2e9fc9b52695b Mon Sep 17 00:00:00 2001 From: Richard Mahn Date: Wed, 6 Feb 2019 11:19:26 -0700 Subject: [PATCH] Feature - Pagination for git tree API (#5838) * Feature - Pagination for git tree API * Handles case when page is negative * Does a for loop over the start and end rather than all entries * Removed redundent logic * Adds per_page as a query parameter * Adds DEFAULT_GIT_TREES_PER_PAGE for settings, ran make fmt * Fix typo in cheat-sheet en * Makes page start at 1, generated swagger * Use updates to SDK * Updates to use latest sdk * Updates swagger for tree api * Adds test for GetTreeBySHA * Updates per PR reviews * Updates per PR reviews * Remove file * Formatting * Fix to swagger file * Fix to swagger * Update v1_json.tmpl * Fix to swagger file --- Gopkg.lock | 4 +- custom/conf/app.ini.sample | 2 + .../doc/advanced/config-cheat-sheet.en-us.md | 1 + .../doc/advanced/config-cheat-sheet.zh-cn.md | 1 + modules/setting/setting.go | 14 ++-- routers/api/v1/repo/tree.go | 68 +++++++++++++------ routers/api/v1/repo/tree_test.go | 48 +++++++++++++ templates/swagger/v1_json.tmpl | 28 ++++++++ vendor/code.gitea.io/sdk/gitea/repo_tree.go | 10 +-- 9 files changed, 145 insertions(+), 31 deletions(-) create mode 100644 routers/api/v1/repo/tree_test.go diff --git a/Gopkg.lock b/Gopkg.lock index 8eb150bc9..2b05b2181 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -11,11 +11,11 @@ [[projects]] branch = "master" - digest = "1:8df1f0527f30a02b76d0ac397118d71c0e9093c7dfa59723762a88fce6ac1170" + digest = "1:17c6c3f4af27f721e3176aceeb2ee30621547a44c81ada0ce733170b9bdfee19" name = "code.gitea.io/sdk" packages = ["gitea"] pruneopts = "NUT" - revision = "d5a42771e7e851e8a89c5c6ffa0f5b075342f9df" + revision = "b9e72373fbe3001d98ce7395221d0134b9456679" [[projects]] digest = "1:5d72bbcc9c8667b11c3dc3cbe681c5a6f71e5096744c0bf7726ab5c6425d5dc4" diff --git a/custom/conf/app.ini.sample b/custom/conf/app.ini.sample index 31fed0c02..126f2a848 100644 --- a/custom/conf/app.ini.sample +++ b/custom/conf/app.ini.sample @@ -629,6 +629,8 @@ ENABLE_SWAGGER = true MAX_RESPONSE_ITEMS = 50 ; Default paging number of api DEFAULT_PAGING_NUM = 30 +; Default and maximum number of items per page for git trees api +DEFAULT_GIT_TREES_PER_PAGE = 1000 [i18n] LANGS = en-US,zh-CN,zh-HK,zh-TW,de-DE,fr-FR,nl-NL,lv-LV,ru-RU,uk-UA,ja-JP,es-ES,pt-BR,pl-PL,bg-BG,it-IT,fi-FI,tr-TR,cs-CZ,sr-SP,sv-SE,ko-KR diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index 89eb9af78..3494311de 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -332,6 +332,7 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`. - `ENABLE_SWAGGER`: **true**: Enables /api/swagger, /api/v1/swagger etc. endpoints. True or false; default is true. - `MAX_RESPONSE_ITEMS`: **50**: Max number of items in a page. - `DEFAULT_PAGING_NUM`: **30**: Default paging number of api. +- `DEFAULT_GIT_TREES_PER_PAGE`: **1000**: Default and maximum number of items per page for git trees api. ## i18n (`i18n`) diff --git a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md index 990d4d42f..2f0251351 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md +++ b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md @@ -199,6 +199,7 @@ menu: - `ENABLE_SWAGGER`: **true**: 是否启用swagger路由 /api/swagger, /api/v1/swagger etc. endpoints. True 或 false; 默认是 true. - `MAX_RESPONSE_ITEMS`: **50**: 一个页面最大的项目数。 - `DEFAULT_PAGING_NUM`: **30**: API中默认分页条数。 +- `DEFAULT_GIT_TREES_PER_PAGE`: **1000**: GIT TREES API每页的默认和最大项数. ## Markup (`markup`) diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 0859b81c4..42f1de425 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -561,13 +561,15 @@ var ( // API settings API = struct { - EnableSwagger bool - MaxResponseItems int - DefaultPagingNum int + EnableSwagger bool + MaxResponseItems int + DefaultPagingNum int + DefaultGitTreesPerPage int }{ - EnableSwagger: true, - MaxResponseItems: 50, - DefaultPagingNum: 30, + EnableSwagger: true, + MaxResponseItems: 50, + DefaultPagingNum: 30, + DefaultGitTreesPerPage: 1000, } U2F = struct { diff --git a/routers/api/v1/repo/tree.go b/routers/api/v1/repo/tree.go index 7288d6cae..8a5d0c4b0 100644 --- a/routers/api/v1/repo/tree.go +++ b/routers/api/v1/repo/tree.go @@ -37,19 +37,34 @@ func GetTree(ctx *context.APIContext) { // description: sha of the commit // type: string // required: true + // - name: recursive + // in: query + // description: show all directories and files + // required: false + // type: boolean + // - name: page + // in: query + // description: page number; the 'truncated' field in the response will be true if there are still more items after this page, false if the last page + // required: false + // type: integer + // - name: per_page + // in: query + // description: number of items per page; default is 1000 or what is set in app.ini as DEFAULT_GIT_TREES_PER_PAGE + // required: false + // type: integer // responses: // "200": // "$ref": "#/responses/GitTreeResponse" sha := ctx.Params("sha") if len(sha) == 0 { - ctx.Error(400, "sha not provided", nil) + ctx.Error(400, "", "sha not provided") return } tree := GetTreeBySHA(ctx, sha) if tree != nil { ctx.JSON(200, tree) } else { - ctx.Error(400, "sha invalid", nil) + ctx.Error(400, "", "sha invalid") } } @@ -87,29 +102,44 @@ func GetTreeBySHA(ctx *context.APIContext, sha string) *gitea.GitTreeResponse { // 40 is the size of the sha1 hash in hexadecimal format. copyPos := len(treeURL) - 40 - if len(entries) > 1000 { - tree.Entries = make([]gitea.GitEntry, 1000) - } else { - tree.Entries = make([]gitea.GitEntry, len(entries)) + page := ctx.QueryInt("page") + perPage := ctx.QueryInt("per_page") + if perPage <= 0 || perPage > setting.API.DefaultGitTreesPerPage { + perPage = setting.API.DefaultGitTreesPerPage } - for e := range entries { - if e > 1000 { - tree.Truncated = true - break - } - - tree.Entries[e].Path = entries[e].Name() - tree.Entries[e].Mode = fmt.Sprintf("%06x", entries[e].Mode()) - tree.Entries[e].Type = string(entries[e].Type) - tree.Entries[e].Size = entries[e].Size() - tree.Entries[e].SHA = entries[e].ID.String() + if page <= 0 { + page = 1 + } + tree.Page = page + tree.TotalCount = len(entries) + rangeStart := perPage * (page - 1) + if rangeStart >= len(entries) { + return tree + } + var rangeEnd int + if len(entries) > perPage { + tree.Truncated = true + } + if rangeStart+perPage < len(entries) { + rangeEnd = rangeStart + perPage + } else { + rangeEnd = len(entries) + } + tree.Entries = make([]gitea.GitEntry, rangeEnd-rangeStart) + for e := rangeStart; e < rangeEnd; e++ { + i := e - rangeStart + tree.Entries[i].Path = entries[e].Name() + tree.Entries[i].Mode = fmt.Sprintf("%06x", entries[e].Mode()) + tree.Entries[i].Type = string(entries[e].Type) + tree.Entries[i].Size = entries[e].Size() + tree.Entries[i].SHA = entries[e].ID.String() if entries[e].IsDir() { copy(treeURL[copyPos:], entries[e].ID.String()) - tree.Entries[e].URL = string(treeURL[:]) + tree.Entries[i].URL = string(treeURL[:]) } else { copy(blobURL[copyPos:], entries[e].ID.String()) - tree.Entries[e].URL = string(blobURL[:]) + tree.Entries[i].URL = string(blobURL[:]) } } return tree diff --git a/routers/api/v1/repo/tree_test.go b/routers/api/v1/repo/tree_test.go new file mode 100644 index 000000000..708516e97 --- /dev/null +++ b/routers/api/v1/repo/tree_test.go @@ -0,0 +1,48 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package repo + +import ( + "github.com/stretchr/testify/assert" + "testing" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/test" + "code.gitea.io/sdk/gitea" +) + +func TestGetTreeBySHA(t *testing.T) { + models.PrepareTestEnv(t) + sha := "master" + ctx := test.MockContext(t, "user2/repo1") + ctx.SetParams(":id", "1") + ctx.SetParams(":sha", sha) + test.LoadRepo(t, ctx, 1) + test.LoadRepoCommit(t, ctx) + test.LoadUser(t, ctx, 2) + test.LoadGitRepo(t, ctx) + + tree := GetTreeBySHA(&context.APIContext{Context: ctx, Org: nil}, ctx.Params("sha")) + expectedTree := &gitea.GitTreeResponse{ + SHA: "65f1bf27bc3bf70f64657658635e66094edbcb4d", + URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/trees/65f1bf27bc3bf70f64657658635e66094edbcb4d", + Entries: []gitea.GitEntry{ + { + Path: "README.md", + Mode: "100644", + Type: "blob", + Size: 30, + SHA: "4b4851ad51df6a7d9f25c979345979eaeb5b349f", + URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/blobs/4b4851ad51df6a7d9f25c979345979eaeb5b349f", + }, + }, + Truncated: false, + Page: 1, + TotalCount: 1, + } + + assert.EqualValues(t, tree, expectedTree) +} diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index bde496c7f..c2ed1d75b 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -1775,6 +1775,24 @@ "name": "sha", "in": "path", "required": true + }, + { + "type": "boolean", + "description": "show all directories and files", + "name": "recursive", + "in": "query" + }, + { + "type": "integer", + "description": "page number; the 'truncated' field in the response will be true if there are still more items after this page, false if the last page", + "name": "page", + "in": "query" + }, + { + "type": "integer", + "description": "number of items per page; default is 1000 or what is set in app.ini as DEFAULT_GIT_TREES_PER_PAGE", + "name": "per_page", + "in": "query" } ], "responses": { @@ -7352,10 +7370,20 @@ "description": "GitTreeResponse returns a git tree", "type": "object", "properties": { + "page": { + "type": "integer", + "format": "int64", + "x-go-name": "Page" + }, "sha": { "type": "string", "x-go-name": "SHA" }, + "total_count": { + "type": "integer", + "format": "int64", + "x-go-name": "TotalCount" + }, "tree": { "type": "array", "items": { diff --git a/vendor/code.gitea.io/sdk/gitea/repo_tree.go b/vendor/code.gitea.io/sdk/gitea/repo_tree.go index cef3c6467..842ab9b43 100644 --- a/vendor/code.gitea.io/sdk/gitea/repo_tree.go +++ b/vendor/code.gitea.io/sdk/gitea/repo_tree.go @@ -20,10 +20,12 @@ type GitEntry struct { // GitTreeResponse returns a git tree type GitTreeResponse struct { - SHA string `json:"sha"` - URL string `json:"url"` - Entries []GitEntry `json:"tree"` - Truncated bool `json:"truncated"` + SHA string `json:"sha"` + URL string `json:"url"` + Entries []GitEntry `json:"tree"` + Truncated bool `json:"truncated"` + Page int `json:"page"` + TotalCount int `json:"total_count"` } // GetTrees downloads a file of repository, ref can be branch/tag/commit.