package main import ( "bitbucket.org/skobkin/point-api-go" "bitbucket.org/skobkin/point-tools-go" "gopkg.in/alecthomas/kingpin.v2" "fmt" "log" "strconv" "time" ) var ( pointApiUrl = kingpin.Flag("point-api", "Point.im API URL").Default("https://point.im/api/").String() pointToolsApiUrl = kingpin.Flag("point-tools-api", "Point Tools API URL").Default("https://point.skobk.in/api/crawler/").String() pointLogin = kingpin.Flag("login", "Point.im login").Short('l').Required().String() pointPassword = kingpin.Flag("password", "Point.im password").Short('p').Required().String() pointToolsToken = kingpin.Flag("point-tools-token", "Point Tools secret token for crawler").Short('t').Required().String() forceContinue = kingpin.Flag("force-continue", "Keep going even after API rejected new page").Short('f').Bool() fromUid = kingpin.Flag("from-uid", "Start after provided post UID").Short('u').Default("0").Int() limit = kingpin.Flag("limit", "How many pages to get").Short('c').Default("0").Int() ) func main() { kingpin.Parse() pointClient := point.New(*pointApiUrl) pointToolsClient := point_tools.New(*pointToolsApiUrl, *pointToolsToken) _, loginErr := pointClient.Login(*pointLogin, *pointPassword) if loginErr != nil { fmt.Printf("Login error %s", loginErr) return } else { fmt.Printf("Successfully authenticated!\n") } var page point.Page var reqErr error = nil if 0 != *fromUid { page, reqErr = pointClient.GetNextAllPostsPageBeforeUid(*fromUid) } else { page, reqErr = pointClient.GetRecentAllPostsPage() } if reqErr != nil { log.Fatal(reqErr) return } fmt.Printf("1 page requested\n") if len(page.Posts) > 0 { fmt.Println("Last uid", strconv.Itoa(page.Posts[len(page.Posts)-1].Uid)) } sendResp, sendErr := pointToolsClient.SendAllPage(page) if sendErr != nil { log.Fatal(sendErr) } if point_tools.STATUS_SUCCESS != sendResp.Status { fmt.Println("Request error", sendResp.Error.Message) } if false == sendResp.Data.Continue && false == *forceContinue { fmt.Println("API rejected next page request") fmt.Println("Exiting.") return } pageNumber := 1 for page.HasNext { pageNumber++ if *limit > 0 && pageNumber > *limit { fmt.Println("Limit reached") break } page, reqErr = pointClient.GetNextAllPostsPage(page) if reqErr != nil { log.Fatal(reqErr) return } fmt.Printf("%d page requested", pageNumber) if len(page.Posts) > 0 { fmt.Printf(", last uid %d", page.Posts[len(page.Posts)-1].Uid) } fmt.Printf(" -> %d posts\n", len(page.Posts)) sendResp, sendErr = pointToolsClient.SendAllPage(page) if sendErr != nil { log.Fatal(sendErr) } if point_tools.STATUS_SUCCESS != sendResp.Status { fmt.Println("Request error", sendResp.Error.Message) } if false == sendResp.Data.Continue && false == *forceContinue { fmt.Println("API rejected next page request") break } time.Sleep(time.Second) } fmt.Println("Exiting.") }