commit 7f3b26a9bb5f57d7fdc9efbef6e2e4d975efb56a from: Sergey Bronnikov via: Sergey Bronnikov date: Fri Mar 26 16:25:24 2021 UTC cmd: limit a number of processed builds Closes #26 commit - da5cc061d5be0b1d4454b02e0d2ace980708f466 commit + 7f3b26a9bb5f57d7fdc9efbef6e2e4d975efb56a blob - 79e8b10a05801db5d59d6b5720e1f783c877e0bd blob + 6d1b15322ba04d615bed48ee1ac1e1c7d8ff62f4 --- backends/backend_azure_devops.go +++ backends/backend_azure_devops.go @@ -2,12 +2,12 @@ package backends import ( "context" + "github.com/ligurio/testres-db/formats" "github.com/microsoft/azure-devops-go-api/azuredevops" "github.com/microsoft/azure-devops-go-api/azuredevops/build" "github.com/microsoft/azure-devops-go-api/azuredevops/core" "github.com/microsoft/azure-devops-go-api/azuredevops/pipelines" "github.com/microsoft/azure-devops-go-api/azuredevops/testresults" - "github.com/ligurio/testres-db/formats" "log" "net/http" ) @@ -127,7 +127,7 @@ func getProjectRef(ctx context.Context, connection *az } // Using custom http client: https://github.com/microsoft/azure-devops-go-api/issues/52 -func SyncAzureDevOps(client *http.Client, b *Backend) (*[]formats.TestResult, error) { +func SyncAzureDevOps(client *http.Client, b *Backend, buildsNumber int) (*[]formats.TestResult, error) { if b.Username != "" { log.Println("Username is specified but unused", b.Username) } blob - 25c002da9397bec11d4dcccffa1cad4e0e984d97 blob + 2e5830ce62dcddf6af043f0a0702a03b8d3b526c --- backends/backend_circleci.go +++ backends/backend_circleci.go @@ -11,12 +11,9 @@ import ( "strings" ) -// limit a number of builds that should be processed -// -1 means all available builds -const buildsNumber = 10 const isDebug = false -func SyncCircleCI(client *http.Client, b *Backend) (*[]formats.TestResult, error) { +func SyncCircleCI(client *http.Client, b *Backend, buildsNumber int) (*[]formats.TestResult, error) { project_path := strings.Split(b.Project, "/") if len(project_path) != 2 { log.Println("Perhaps wrong project name specified") @@ -45,6 +42,9 @@ func SyncCircleCI(client *http.Client, b *Backend) (*[ "failed": formats.StatusFail, } + if buildsNumber != -1 && len(builds) > buildsNumber { + builds = builds[:buildsNumber] + } results := make([]formats.TestResult, len(builds)) for _, build := range builds { log.Printf("Found build: %d, status %s\n", build.BuildNum, build.Status) blob - 3720381818e2409b89936a7a492a51e4fd38d367 blob + ee0af2d0bdae5063d0237eaf0adcd8ac75f1dda4 --- backends/backend_circleci_test.go +++ backends/backend_circleci_test.go @@ -15,8 +15,9 @@ func TestSyncCircleCI(t *testing.T) { backend := Backend{Type: "circleci", Base: "https://circleci.com/", Name: "spotify/helios", Project: "spotify/helios", Branch: "master", Username: username, Secret: token} + buildsNumber := 5 httpClient := NewAPIClient() - builds, err := SyncCircleCI(httpClient, &backend) + builds, err := SyncCircleCI(httpClient, &backend, buildsNumber) if builds == nil || err != nil { t.Failed() } blob - b8a4f49dd4208ff17e87a571d3c1476e4fef540e blob + 88271c70cdf0b635ac903877e59e89f428acb8cf --- backends/backend_cirrusci.go +++ backends/backend_cirrusci.go @@ -22,13 +22,13 @@ https://api.cirrus-ci.com/graphql | python -m json.too package backends import ( - "github.com/machinebox/graphql" "github.com/ligurio/testres-db/formats" + "github.com/machinebox/graphql" "golang.org/x/net/context" "net/http" ) -func SyncCirrusCI(client *http.Client, b *Backend) (*[]formats.TestResult, error) { +func SyncCirrusCI(client *http.Client, b *Backend, buildsNumber int) (*[]formats.TestResult, error) { graphql_scheme := "https://api.cirrus-ci.com/graphql" ClientOption := graphql.WithHTTPClient(client) connection := graphql.NewClient(graphql_scheme, ClientOption) blob - 0e2d89eb711612206f1df7348fa41e333d7aad21 blob + 61ba3431eeb99bd7014dd28eb6e30cd21a027f39 --- backends/backend_common.go +++ backends/backend_common.go @@ -10,7 +10,7 @@ import ( "os" ) -type fnSyncBackend func(client *http.Client, b *Backend) (*[]formats.TestResult, error) +type fnSyncBackend func(client *http.Client, b *Backend, buildsNumber int) (*[]formats.TestResult, error) var backend = map[string]fnSyncBackend{ "azure_devops": SyncAzureDevOps, @@ -52,14 +52,14 @@ func NewAPIClient() *http.Client { return client } -func (b *Backend) GetTestResults() (*[]formats.TestResult, error) { +func (b *Backend) GetTestResults(buildsNumber int) (*[]formats.TestResult, error) { log.Println("Backend:", b.Type) fn := backend[b.Type] if fn == nil { return nil, errUnknownBackend } client := NewAPIClient() - result, err := fn(client, b) + result, err := fn(client, b, buildsNumber) if err != nil { return nil, err } blob - 4b5e3cc92bae56f0b272ff1a32786e4a306c6474 blob + d1b92b1967cdbe8c132411d5db1edb67f1050138 --- backends/backend_gitlab.go +++ backends/backend_gitlab.go @@ -9,7 +9,7 @@ import ( "path/filepath" ) -func SyncGitLab(client *http.Client, b *Backend) (*[]formats.TestResult, error) { +func SyncGitLab(client *http.Client, b *Backend, buildsNumber int) (*[]formats.TestResult, error) { if b.Pipeline != "" { log.Println("Option pipeline is specified, but unused") } @@ -60,6 +60,9 @@ func SyncGitLab(client *http.Client, b *Backend) (*[]f ListOptions: gitlab.ListOptions{Page: 1, PerPage: 10}, Scope: []gitlab.BuildStateValue{"created", "pending", "running", "failed", "success", "canceled", "skipped"}, } + if buildsNumber != -1 && len(pipelines) > buildsNumber { + pipelines = pipelines[:buildsNumber] + } for _, pipeline := range pipelines { log.Printf("Found pipeline: %d, status %s", pipeline.ID, pipeline.Status) log.Printf("SHA %s, Ref %s", pipeline.SHA, pipeline.Ref) blob - 766ae18bbe8413baa7ddfb361224a282e2e63243 blob + bd8ebd4b3289e5022d7a62def8bb7d69a060d728 --- backends/backend_jenkins.go +++ backends/backend_jenkins.go @@ -8,7 +8,7 @@ import ( "net/http" ) -func SyncJenkins(client *http.Client, b *Backend) (*[]formats.TestResult, error) { +func SyncJenkins(client *http.Client, b *Backend, buildsNumber int) (*[]formats.TestResult, error) { var jenkins *gojenkins.Jenkins jenkins = gojenkins.CreateJenkins(client, b.Base, b.Username, b.Secret) ctx := context.Background() @@ -22,6 +22,9 @@ func SyncJenkins(client *http.Client, b *Backend) (*[] return nil, err } + if buildsNumber != -1 && len(jobBuilds) > buildsNumber { + jobBuilds = jobBuilds[:buildsNumber] + } results := make([]formats.TestResult, len(jobBuilds)) for _, jobBuild := range jobBuilds { buildNum, err := jenkins.GetBuild(ctx, b.Pipeline, jobBuild.Number) blob - c4e4e043f581325d356cf695e7824fe7d5f735a5 blob + 7f2677c6bd85ece51c614dbb8d5a86dd2ad4899e --- backends/backend_teamcity.go +++ backends/backend_teamcity.go @@ -10,7 +10,7 @@ import ( "net/http" ) -func SyncTeamCity(client *http.Client, b *Backend) (*[]formats.TestResult, error) { +func SyncTeamCity(client *http.Client, b *Backend, buildsNumber int) (*[]formats.TestResult, error) { connection, err := teamcity.NewWithAddress(b.Username, b.Secret, b.Base, client) if err != nil { return nil, err blob - 695ea07b5ddc4358ce4b62f515533e69bda9e3d7 blob + 5d6764261774b98859181939b37929e8fe387d7e --- backends/backend_travisci.go +++ backends/backend_travisci.go @@ -10,7 +10,7 @@ import ( "path" ) -func SyncTravisCI(client *http.Client, b *Backend) (*[]formats.TestResult, error) { +func SyncTravisCI(client *http.Client, b *Backend, buildsNumber int) (*[]formats.TestResult, error) { connection := travisci.NewClient(b.Base, b.Secret) connection.HTTPClient = client build_service := connection.Builds @@ -24,6 +24,9 @@ func SyncTravisCI(client *http.Client, b *Backend) (*[ results := make([]formats.TestResult, len(builds)) baseURL := "https://travis-ci.org/" + if buildsNumber != -1 && len(builds) > buildsNumber { + builds = builds[:buildsNumber] + } for _, build := range builds { // https://godoc.org/github.com/shuheiktgw/go-travis#Build metadata := *build.Metadata blob - 3b168eefe326c0ba6a9bca24f3f8f73579cdfaa4 blob + 93f7971b3f57ba4a558534c48734c01916f2e80d --- cmd/testres-db/testres-db.go +++ cmd/testres-db/testres-db.go @@ -15,12 +15,14 @@ var options struct { configName string dbName string projectName string + buildsNumber int } func init() { flag.StringVar(&options.configName, "config", "testres-db.yaml", "config") flag.StringVar(&options.dbName, "db", "testres.sqlite", "database file") flag.StringVar(&options.projectName, "project", "", "project name") + flag.IntVar(&options.buildsNumber, "limit", -1, "limit a number of builds") flag.Usage = func() { fmt.Fprintf(os.Stderr, "Import test results to an SQLite DB.\n\n") @@ -91,7 +93,7 @@ func main() { if options.projectName != "" { if options.projectName == p.Name { for _, b := range p.Backends { - results, err := b.GetTestResults() + results, err := b.GetTestResults(options.buildsNumber) if err != nil { log.Println(err) continue @@ -107,7 +109,7 @@ func main() { } } else { for _, b := range p.Backends { - results, err := b.GetTestResults() + results, err := b.GetTestResults(options.buildsNumber) if err != nil { log.Println(err) continue