diff --git a/point_post_crawler.go b/point_post_crawler.go index 153de3a..cdc6a91 100644 --- a/point_post_crawler.go +++ b/point_post_crawler.go @@ -7,6 +7,7 @@ import ( "bitbucket.org/skobkin/point-tools-crawler/point_tools" "log" "time" + "strconv" ) func main() { @@ -24,53 +25,73 @@ func main() { pointClient := point.NewClient("https://point.im/api/") pointToolsClient := point_tools.NewClient("http://point-tools.local:8000/api/crawler/") - _, login_err := pointClient.Login(login, password) + _, loginErr := pointClient.Login(login, password) - if login_err != nil { - fmt.Printf("Login error %s", login_err) + if loginErr != nil { + fmt.Printf("Login error %s", loginErr) return } else { fmt.Printf("Successfully authenticated!\n") } - page, req_err := pointClient.GetRecentAllPostsPage() + page, reqErr := pointClient.GetRecentAllPostsPage() - if req_err != nil { - log.Fatal(req_err) + if reqErr != nil { + log.Fatal(reqErr) return } fmt.Printf("1 page requested\n") - if len(page.Posts) == 0 { - fmt.Println("Last uid", string(page.Posts[len(page.Posts)-1].Uid)) + if len(page.Posts) > 0 { + fmt.Println("Last uid", strconv.Itoa(page.Posts[len(page.Posts)-1].Uid)) } - pointToolsClient.SendPage(page) + sendResp, sendErr := pointToolsClient.SendPage(page) - //var str string + if sendErr != nil { + log.Fatal(sendErr) + } + + if point_tools.STATUS_SUCCESS != sendResp.Status || false == sendResp.Data.Continue { + fmt.Println("API rejected next page request") + fmt.Println("Exiting.") + return + } pageNumber := 1 for page.HasNext { pageNumber++ - page, req_err = pointClient.GetNextAllPostsPage(page) + page, reqErr = pointClient.GetNextAllPostsPage(page) - if req_err != nil { - log.Fatal(req_err) + if reqErr != nil { + log.Fatal(reqErr) return } + fmt.Printf("%d page requested", pageNumber) + if len(page.Posts) > 0 { + fmt.Printf(", last uid %d", page.Posts[len(page.Posts)-1].Uid) + } + fmt.Printf(" -> %d posts\n", len(page.Posts)) + sendResp, sendErr := pointToolsClient.SendPage(page) - fmt.Printf("%d page requested, last uid: %d", pageNumber) - fmt.Printf("%d posts\n", len(page.Posts)) + if sendErr != nil { + log.Fatal(sendErr) + } - pointToolsClient.SendPage(page) + if point_tools.STATUS_SUCCESS != sendResp.Status || false == sendResp.Data.Continue { + fmt.Println("API rejected next page request") + break + } time.Sleep(time.Second) } + + fmt.Println("Exiting.") } diff --git a/point_tools/client.go b/point_tools/client.go index 9af246d..3969364 100644 --- a/point_tools/client.go +++ b/point_tools/client.go @@ -57,7 +57,5 @@ func (c *PointToolsClient) SendPage(page point.Page) (ReceiveAllPageResponse, er return response, json_err } - log.Println("Response:", string(body[:])) - return response, nil } \ No newline at end of file diff --git a/point_tools/receive_all_page_response.go b/point_tools/receive_all_page_response.go index b8c6ee0..fa73554 100644 --- a/point_tools/receive_all_page_response.go +++ b/point_tools/receive_all_page_response.go @@ -1,5 +1,10 @@ package point_tools +const ( + STATUS_SUCCESS string = "success" + STATUS_FAILURE string = "fail" +) + type ReceiveAllPageResponse struct { Status string `json:"status"` Data ReceiveAllPageData `json:"data"`