package main import ( "bitbucket.org/skobkin/point-tools-crawler/point" "bitbucket.org/skobkin/point-tools-crawler/point_tools" "flag" "fmt" "log" "strconv" "time" ) func main() { var pointApiUrl, pointToolsApiUrl string; var pointLogin, pointPassword, pointToolsToken string var forceContinue bool var fromUid, limit int // Todo refactor the CLI options to new library flag.StringVar(&pointLogin, "l", "", "Account login") flag.StringVar(&pointPassword, "p", "", "Account password") flag.StringVar(&pointToolsToken, "t", "", "Point Tools crawler API token") flag.StringVar(&pointApiUrl, "s", "https://point.im/api/", "Point.im API url") flag.StringVar(&pointToolsApiUrl, "g", "https://point.skobk.in/api/crawler/", "Point Tools API url") flag.BoolVar(&forceContinue, "f", false, "Force continue reading /all despite of server refusal") flag.IntVar(&fromUid, "u", 0, "From which UID start to load pages") flag.IntVar(&limit, "c", 0, "How many pages to get") flag.Parse() if len(pointLogin) < 1 || len(pointPassword) < 1 { fmt.Println("Login and password must be defined") return } if len(pointToolsToken) < 1 { fmt.Println("Point Tools token must be defined") return } pointClient := point.NewClient(pointApiUrl) pointToolsClient := point_tools.NewClient(pointToolsApiUrl, pointToolsToken) _, loginErr := pointClient.Login(pointLogin, pointPassword) if loginErr != nil { fmt.Printf("Login error %s", loginErr) return } else { fmt.Printf("Successfully authenticated!\n") } var page point.Page var reqErr error = nil if 0 != fromUid { page, reqErr = pointClient.GetNextAllPostsPageBeforeUid(fromUid) } else { page, reqErr = pointClient.GetRecentAllPostsPage() } if reqErr != nil { log.Fatal(reqErr) return } fmt.Printf("1 page requested\n") if len(page.Posts) > 0 { fmt.Println("Last uid", strconv.Itoa(page.Posts[len(page.Posts)-1].Uid)) } sendResp, sendErr := pointToolsClient.SendPage(page) if sendErr != nil { log.Fatal(sendErr) } if point_tools.STATUS_SUCCESS != sendResp.Status { fmt.Println("Request error", sendResp.Error.Message) } if false == sendResp.Data.Continue && false == forceContinue { fmt.Println("API rejected next page request") fmt.Println("Exiting.") return } pageNumber := 1 for page.HasNext { pageNumber++ if limit > 0 && pageNumber > limit { fmt.Println("Limit reached") break } page, reqErr = pointClient.GetNextAllPostsPage(page) if reqErr != nil { log.Fatal(reqErr) return } fmt.Printf("%d page requested", pageNumber) if len(page.Posts) > 0 { fmt.Printf(", last uid %d", page.Posts[len(page.Posts)-1].Uid) } fmt.Printf(" -> %d posts\n", len(page.Posts)) sendResp, sendErr = pointToolsClient.SendPage(page) if sendErr != nil { log.Fatal(sendErr) } if point_tools.STATUS_SUCCESS != sendResp.Status { fmt.Println("Request error", sendResp.Error.Message) } if false == sendResp.Data.Continue && false == forceContinue { fmt.Println("API rejected next page request") break } time.Sleep(time.Second) } fmt.Println("Exiting.") }