skobkin/point-tools-crawler
1
0
Fork 0
mirror of https://bitbucket.org/skobkin/point-tools-crawler.git synced 2024-12-05 02:25:53 +00:00
Code Issues Projects Releases Wiki Activity

-f flag added (force continue). Error message when response has error.

Browse source
This commit is contained in:
Alexey Skobkin 2016-03-25 20:12:13 +03:00
parent f1d46ca56c
commit d791b2377c
1 changed files with 13 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
point_post_crawler.go
View file

@ -13,12 +13,14 @@ import (
func main() { func main() {
var pointApiUrl, pointToolsApiUrl string; var pointApiUrl, pointToolsApiUrl string;
var pointLogin, pointPassword, pointToolsToken string var pointLogin, pointPassword, pointToolsToken string
var forceContinue bool
flag.StringVar(&pointLogin, "l", "", "Account login") flag.StringVar(&pointLogin, "l", "", "Account login")
flag.StringVar(&pointPassword, "p", "", "Account password") flag.StringVar(&pointPassword, "p", "", "Account password")
flag.StringVar(&pointToolsToken, "t", "", "Point Tools crawler API token") flag.StringVar(&pointToolsToken, "t", "", "Point Tools crawler API token")
flag.StringVar(&pointApiUrl, "s", "https://point.im/api/", "Point.im API url") flag.StringVar(&pointApiUrl, "s", "https://point.im/api/", "Point.im API url")
flag.StringVar(&pointToolsApiUrl, "g", "https://point.skobk.in/api/crawler/", "Point Tools API url") flag.StringVar(&pointToolsApiUrl, "g", "https://point.skobk.in/api/crawler/", "Point Tools API url")
flag.BoolVar(&forceContinue, "f", false, "Force continue reading /all despite of server refusal")
flag.Parse() flag.Parse()
if len(pointLogin) < 1 || len(pointPassword) < 1 { if len(pointLogin) < 1 || len(pointPassword) < 1 {
@ -62,7 +64,11 @@ func main() {
log.Fatal(sendErr) log.Fatal(sendErr)
} }
if point_tools.STATUS_SUCCESS != sendResp.Status || false == sendResp.Data.Continue { if point_tools.STATUS_SUCCESS != sendResp.Status {
fmt.Println("Request error", sendResp.Error.Message)
}
if false == sendResp.Data.Continue && false == forceContinue {
fmt.Println("API rejected next page request") fmt.Println("API rejected next page request")
fmt.Println("Exiting.") fmt.Println("Exiting.")
return return
@ -86,13 +92,17 @@ func main() {
} }
fmt.Printf(" -> %d posts\n", len(page.Posts)) fmt.Printf(" -> %d posts\n", len(page.Posts))
sendResp, sendErr := pointToolsClient.SendPage(page) sendResp, sendErr = pointToolsClient.SendPage(page)
if sendErr != nil { if sendErr != nil {
log.Fatal(sendErr) log.Fatal(sendErr)
} }
if point_tools.STATUS_SUCCESS != sendResp.Status || false == sendResp.Data.Continue { if point_tools.STATUS_SUCCESS != sendResp.Status {
fmt.Println("Request error", sendResp.Error.Message)
}
if false == sendResp.Data.Continue && false == forceContinue {
fmt.Println("API rejected next page request") fmt.Println("API rejected next page request")
break break
} }