// Copyright 2021 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT package migrations import ( "context" "encoding/xml" "fmt" "net/http" "net/url" "strconv" "strings" "time" "code.gitea.io/gitea/modules/log" base "code.gitea.io/gitea/modules/migration" "code.gitea.io/gitea/modules/proxy" "code.gitea.io/gitea/modules/structs" ) var ( _ base.Downloader = &CodebaseDownloader{} _ base.DownloaderFactory = &CodebaseDownloaderFactory{} ) func init() { RegisterDownloaderFactory(&CodebaseDownloaderFactory{}) } // CodebaseDownloaderFactory defines a downloader factory type CodebaseDownloaderFactory struct{} // New returns a downloader related to this factory according MigrateOptions func (f *CodebaseDownloaderFactory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) { u, err := url.Parse(opts.CloneAddr) if err != nil { return nil, err } u.User = nil fields := strings.Split(strings.Trim(u.Path, "/"), "/") if len(fields) != 2 { return nil, fmt.Errorf("invalid path: %s", u.Path) } project := fields[0] repoName := strings.TrimSuffix(fields[1], ".git") log.Trace("Create Codebase downloader. BaseURL: %v RepoName: %s", u, repoName) return NewCodebaseDownloader(ctx, u, project, repoName, opts.AuthUsername, opts.AuthPassword), nil } // GitServiceType returns the type of git service func (f *CodebaseDownloaderFactory) GitServiceType() structs.GitServiceType { return structs.CodebaseService } type codebaseUser struct { ID int64 `json:"id"` Name string `json:"name"` Email string `json:"email"` } // CodebaseDownloader implements a Downloader interface to get repository information // from Codebase type CodebaseDownloader struct { base.NullDownloader ctx context.Context client *http.Client baseURL *url.URL projectURL *url.URL project string repoName string maxIssueIndex int64 userMap map[int64]*codebaseUser commitMap map[string]string } // SetContext set context func (d *CodebaseDownloader) SetContext(ctx context.Context) { d.ctx = ctx } // NewCodebaseDownloader creates a new downloader func NewCodebaseDownloader(ctx context.Context, projectURL *url.URL, project, repoName, username, password string) *CodebaseDownloader { baseURL, _ := url.Parse("https://api3.codebasehq.com") downloader := &CodebaseDownloader{ ctx: ctx, baseURL: baseURL, projectURL: projectURL, project: project, repoName: repoName, client: &http.Client{ Transport: &http.Transport{ Proxy: func(req *http.Request) (*url.URL, error) { if len(username) > 0 && len(password) > 0 { req.SetBasicAuth(username, password) } return proxy.Proxy()(req) }, }, }, userMap: make(map[int64]*codebaseUser), commitMap: make(map[string]string), } log.Trace("Create Codebase downloader. BaseURL: %s Project: %s RepoName: %s", baseURL, project, repoName) return downloader } // String implements Stringer func (d *CodebaseDownloader) String() string { return fmt.Sprintf("migration from codebase server %s %s/%s", d.baseURL, d.project, d.repoName) } // ColorFormat provides a basic color format for a GogsDownloader func (d *CodebaseDownloader) ColorFormat(s fmt.State) { if d == nil { log.ColorFprintf(s, "<nil: CodebaseDownloader>") return } log.ColorFprintf(s, "migration from codebase server %s %s/%s", d.baseURL, d.project, d.repoName) } // FormatCloneURL add authentication into remote URLs func (d *CodebaseDownloader) FormatCloneURL(opts base.MigrateOptions, remoteAddr string) (string, error) { return opts.CloneAddr, nil } func (d *CodebaseDownloader) callAPI(endpoint string, parameter map[string]string, result interface{}) error { u, err := d.baseURL.Parse(endpoint) if err != nil { return err } if parameter != nil { query := u.Query() for k, v := range parameter { query.Set(k, v) } u.RawQuery = query.Encode() } req, err := http.NewRequestWithContext(d.ctx, "GET", u.String(), nil) if err != nil { return err } req.Header.Add("Accept", "application/xml") resp, err := d.client.Do(req) if err != nil { return err } defer resp.Body.Close() return xml.NewDecoder(resp.Body).Decode(&result) } // GetRepoInfo returns repository information // https://support.codebasehq.com/kb/projects func (d *CodebaseDownloader) GetRepoInfo() (*base.Repository, error) { var rawRepository struct { XMLName xml.Name `xml:"repository"` Name string `xml:"name"` Description string `xml:"description"` Permalink string `xml:"permalink"` CloneURL string `xml:"clone-url"` Source string `xml:"source"` } err := d.callAPI( fmt.Sprintf("/%s/%s", d.project, d.repoName), nil, &rawRepository, ) if err != nil { return nil, err } return &base.Repository{ Name: rawRepository.Name, Description: rawRepository.Description, CloneURL: rawRepository.CloneURL, OriginalURL: d.projectURL.String(), }, nil } // GetMilestones returns milestones // https://support.codebasehq.com/kb/tickets-and-milestones/milestones func (d *CodebaseDownloader) GetMilestones() ([]*base.Milestone, error) { var rawMilestones struct { XMLName xml.Name `xml:"ticketing-milestone"` Type string `xml:"type,attr"` TicketingMilestone []struct { Text string `xml:",chardata"` ID struct { Value int64 `xml:",chardata"` Type string `xml:"type,attr"` } `xml:"id"` Identifier string `xml:"identifier"` Name string `xml:"name"` Deadline struct { Value string `xml:",chardata"` Type string `xml:"type,attr"` } `xml:"deadline"` Description string `xml:"description"` Status string `xml:"status"` } `xml:"ticketing-milestone"` } err := d.callAPI( fmt.Sprintf("/%s/milestones", d.project), nil, &rawMilestones, ) if err != nil { return nil, err } milestones := make([]*base.Milestone, 0, len(rawMilestones.TicketingMilestone)) for _, milestone := range rawMilestones.TicketingMilestone { var deadline *time.Time if len(milestone.Deadline.Value) > 0 { if val, err := time.Parse("2006-01-02", milestone.Deadline.Value); err == nil { deadline = &val } } closed := deadline state := "closed" if milestone.Status == "active" { closed = nil state = "" } milestones = append(milestones, &base.Milestone{ Title: milestone.Name, Deadline: deadline, Closed: closed, State: state, }) } return milestones, nil } // GetLabels returns labels // https://support.codebasehq.com/kb/tickets-and-milestones/statuses-priorities-and-categories func (d *CodebaseDownloader) GetLabels() ([]*base.Label, error) { var rawTypes struct { XMLName xml.Name `xml:"ticketing-types"` Type string `xml:"type,attr"` TicketingType []struct { ID struct { Value int64 `xml:",chardata"` Type string `xml:"type,attr"` } `xml:"id"` Name string `xml:"name"` } `xml:"ticketing-type"` } err := d.callAPI( fmt.Sprintf("/%s/tickets/types", d.project), nil, &rawTypes, ) if err != nil { return nil, err } labels := make([]*base.Label, 0, len(rawTypes.TicketingType)) for _, label := range rawTypes.TicketingType { labels = append(labels, &base.Label{ Name: label.Name, Color: "ffffff", }) } return labels, nil } type codebaseIssueContext struct { Comments []*base.Comment } // GetIssues returns issues, limits are not supported // https://support.codebasehq.com/kb/tickets-and-milestones // https://support.codebasehq.com/kb/tickets-and-milestones/updating-tickets func (d *CodebaseDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { var rawIssues struct { XMLName xml.Name `xml:"tickets"` Type string `xml:"type,attr"` Ticket []struct { TicketID struct { Value int64 `xml:",chardata"` Type string `xml:"type,attr"` } `xml:"ticket-id"` Summary string `xml:"summary"` TicketType string `xml:"ticket-type"` ReporterID struct { Value int64 `xml:",chardata"` Type string `xml:"type,attr"` } `xml:"reporter-id"` Reporter string `xml:"reporter"` Type struct { Name string `xml:"name"` } `xml:"type"` Status struct { TreatAsClosed struct { Value bool `xml:",chardata"` Type string `xml:"type,attr"` } `xml:"treat-as-closed"` } `xml:"status"` Milestone struct { Name string `xml:"name"` } `xml:"milestone"` UpdatedAt struct { Value time.Time `xml:",chardata"` Type string `xml:"type,attr"` } `xml:"updated-at"` CreatedAt struct { Value time.Time `xml:",chardata"` Type string `xml:"type,attr"` } `xml:"created-at"` } `xml:"ticket"` } err := d.callAPI( fmt.Sprintf("/%s/tickets", d.project), nil, &rawIssues, ) if err != nil { return nil, false, err } issues := make([]*base.Issue, 0, len(rawIssues.Ticket)) for _, issue := range rawIssues.Ticket { var notes struct { XMLName xml.Name `xml:"ticket-notes"` Type string `xml:"type,attr"` TicketNote []struct { Content string `xml:"content"` CreatedAt struct { Value time.Time `xml:",chardata"` Type string `xml:"type,attr"` } `xml:"created-at"` UpdatedAt struct { Value time.Time `xml:",chardata"` Type string `xml:"type,attr"` } `xml:"updated-at"` ID struct { Value int64 `xml:",chardata"` Type string `xml:"type,attr"` } `xml:"id"` UserID struct { Value int64 `xml:",chardata"` Type string `xml:"type,attr"` } `xml:"user-id"` } `xml:"ticket-note"` } err := d.callAPI( fmt.Sprintf("/%s/tickets/%d/notes", d.project, issue.TicketID.Value), nil, ¬es, ) if err != nil { return nil, false, err } comments := make([]*base.Comment, 0, len(notes.TicketNote)) for _, note := range notes.TicketNote { if len(note.Content) == 0 { continue } poster := d.tryGetUser(note.UserID.Value) comments = append(comments, &base.Comment{ IssueIndex: issue.TicketID.Value, Index: note.ID.Value, PosterID: poster.ID, PosterName: poster.Name, PosterEmail: poster.Email, Content: note.Content, Created: note.CreatedAt.Value, Updated: note.UpdatedAt.Value, }) } if len(comments) == 0 { comments = append(comments, &base.Comment{}) } state := "open" if issue.Status.TreatAsClosed.Value { state = "closed" } poster := d.tryGetUser(issue.ReporterID.Value) issues = append(issues, &base.Issue{ Title: issue.Summary, Number: issue.TicketID.Value, PosterName: poster.Name, PosterEmail: poster.Email, Content: comments[0].Content, Milestone: issue.Milestone.Name, State: state, Created: issue.CreatedAt.Value, Updated: issue.UpdatedAt.Value, Labels: []*base.Label{ {Name: issue.Type.Name}, }, ForeignIndex: issue.TicketID.Value, Context: codebaseIssueContext{ Comments: comments[1:], }, }) if d.maxIssueIndex < issue.TicketID.Value { d.maxIssueIndex = issue.TicketID.Value } } return issues, true, nil } // GetComments returns comments func (d *CodebaseDownloader) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { context, ok := commentable.GetContext().(codebaseIssueContext) if !ok { return nil, false, fmt.Errorf("unexpected context: %+v", commentable.GetContext()) } return context.Comments, true, nil } // GetPullRequests returns pull requests // https://support.codebasehq.com/kb/repositories/merge-requests func (d *CodebaseDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { var rawMergeRequests struct { XMLName xml.Name `xml:"merge-requests"` Type string `xml:"type,attr"` MergeRequest []struct { ID struct { Value int64 `xml:",chardata"` Type string `xml:"type,attr"` } `xml:"id"` } `xml:"merge-request"` } err := d.callAPI( fmt.Sprintf("/%s/%s/merge_requests", d.project, d.repoName), map[string]string{ "query": `"Target Project" is "` + d.repoName + `"`, "offset": strconv.Itoa((page - 1) * perPage), "count": strconv.Itoa(perPage), }, &rawMergeRequests, ) if err != nil { return nil, false, err } pullRequests := make([]*base.PullRequest, 0, len(rawMergeRequests.MergeRequest)) for i, mr := range rawMergeRequests.MergeRequest { var rawMergeRequest struct { XMLName xml.Name `xml:"merge-request"` ID struct { Value int64 `xml:",chardata"` Type string `xml:"type,attr"` } `xml:"id"` SourceRef string `xml:"source-ref"` // NOTE: from the documentation these are actually just branches NOT full refs TargetRef string `xml:"target-ref"` // NOTE: from the documentation these are actually just branches NOT full refs Subject string `xml:"subject"` Status string `xml:"status"` UserID struct { Value int64 `xml:",chardata"` Type string `xml:"type,attr"` } `xml:"user-id"` CreatedAt struct { Value time.Time `xml:",chardata"` Type string `xml:"type,attr"` } `xml:"created-at"` UpdatedAt struct { Value time.Time `xml:",chardata"` Type string `xml:"type,attr"` } `xml:"updated-at"` Comments struct { Type string `xml:"type,attr"` Comment []struct { Content string `xml:"content"` ID struct { Value int64 `xml:",chardata"` Type string `xml:"type,attr"` } `xml:"id"` UserID struct { Value int64 `xml:",chardata"` Type string `xml:"type,attr"` } `xml:"user-id"` Action struct { Value string `xml:",chardata"` Nil string `xml:"nil,attr"` } `xml:"action"` CreatedAt struct { Value time.Time `xml:",chardata"` Type string `xml:"type,attr"` } `xml:"created-at"` } `xml:"comment"` } `xml:"comments"` } err := d.callAPI( fmt.Sprintf("/%s/%s/merge_requests/%d", d.project, d.repoName, mr.ID.Value), nil, &rawMergeRequest, ) if err != nil { return nil, false, err } number := d.maxIssueIndex + int64(i) + 1 state := "open" merged := false var closeTime *time.Time var mergedTime *time.Time if rawMergeRequest.Status != "new" { state = "closed" closeTime = &rawMergeRequest.UpdatedAt.Value } comments := make([]*base.Comment, 0, len(rawMergeRequest.Comments.Comment)) for _, comment := range rawMergeRequest.Comments.Comment { if len(comment.Content) == 0 { if comment.Action.Value == "merging" { merged = true mergedTime = &comment.CreatedAt.Value } continue } poster := d.tryGetUser(comment.UserID.Value) comments = append(comments, &base.Comment{ IssueIndex: number, Index: comment.ID.Value, PosterID: poster.ID, PosterName: poster.Name, PosterEmail: poster.Email, Content: comment.Content, Created: comment.CreatedAt.Value, Updated: comment.CreatedAt.Value, }) } if len(comments) == 0 { comments = append(comments, &base.Comment{}) } poster := d.tryGetUser(rawMergeRequest.UserID.Value) pullRequests = append(pullRequests, &base.PullRequest{ Title: rawMergeRequest.Subject, Number: number, PosterName: poster.Name, PosterEmail: poster.Email, Content: comments[0].Content, State: state, Created: rawMergeRequest.CreatedAt.Value, Updated: rawMergeRequest.UpdatedAt.Value, Closed: closeTime, Merged: merged, MergedTime: mergedTime, Head: base.PullRequestBranch{ Ref: rawMergeRequest.SourceRef, SHA: d.getHeadCommit(rawMergeRequest.SourceRef), RepoName: d.repoName, }, Base: base.PullRequestBranch{ Ref: rawMergeRequest.TargetRef, SHA: d.getHeadCommit(rawMergeRequest.TargetRef), RepoName: d.repoName, }, ForeignIndex: rawMergeRequest.ID.Value, Context: codebaseIssueContext{ Comments: comments[1:], }, }) // SECURITY: Ensure that the PR is safe _ = CheckAndEnsureSafePR(pullRequests[len(pullRequests)-1], d.baseURL.String(), d) } return pullRequests, true, nil } func (d *CodebaseDownloader) tryGetUser(userID int64) *codebaseUser { if len(d.userMap) == 0 { var rawUsers struct { XMLName xml.Name `xml:"users"` Type string `xml:"type,attr"` User []struct { EmailAddress string `xml:"email-address"` ID struct { Value int64 `xml:",chardata"` Type string `xml:"type,attr"` } `xml:"id"` LastName string `xml:"last-name"` FirstName string `xml:"first-name"` Username string `xml:"username"` } `xml:"user"` } err := d.callAPI( "/users", nil, &rawUsers, ) if err == nil { for _, user := range rawUsers.User { d.userMap[user.ID.Value] = &codebaseUser{ Name: user.Username, Email: user.EmailAddress, } } } } user, ok := d.userMap[userID] if !ok { user = &codebaseUser{ Name: fmt.Sprintf("User %d", userID), } d.userMap[userID] = user } return user } func (d *CodebaseDownloader) getHeadCommit(ref string) string { commitRef, ok := d.commitMap[ref] if !ok { var rawCommits struct { XMLName xml.Name `xml:"commits"` Type string `xml:"type,attr"` Commit []struct { Ref string `xml:"ref"` } `xml:"commit"` } err := d.callAPI( fmt.Sprintf("/%s/%s/commits/%s", d.project, d.repoName, ref), nil, &rawCommits, ) if err == nil && len(rawCommits.Commit) > 0 { commitRef = rawCommits.Commit[0].Ref d.commitMap[ref] = commitRef } } return commitRef }