point-tools-crawler/point_post_crawler.go

131 lines
3.1 KiB
Go
Raw Normal View History

2016-03-07 20:04:50 +00:00
package main
import (
"bitbucket.org/skobkin/point-tools-crawler/point"
"bitbucket.org/skobkin/point-tools-crawler/point_tools"
2016-03-21 20:54:09 +00:00
"flag"
"fmt"
2016-03-07 20:04:50 +00:00
"log"
"strconv"
2016-03-21 20:54:09 +00:00
"time"
2016-03-07 20:04:50 +00:00
)
func main() {
2016-03-25 15:23:40 +00:00
var pointApiUrl, pointToolsApiUrl string;
2016-03-22 20:12:37 +00:00
var pointLogin, pointPassword, pointToolsToken string
var forceContinue bool
var fromUid, limit int
2016-03-07 20:04:50 +00:00
// Todo refactor the CLI options to new library
2016-03-22 20:12:37 +00:00
flag.StringVar(&pointLogin, "l", "", "Account login")
flag.StringVar(&pointPassword, "p", "", "Account password")
flag.StringVar(&pointToolsToken, "t", "", "Point Tools crawler API token")
2016-03-25 15:23:40 +00:00
flag.StringVar(&pointApiUrl, "s", "https://point.im/api/", "Point.im API url")
flag.StringVar(&pointToolsApiUrl, "g", "https://point.skobk.in/api/crawler/", "Point Tools API url")
flag.BoolVar(&forceContinue, "f", false, "Force continue reading /all despite of server refusal")
flag.IntVar(&fromUid, "u", 0, "From which UID start to load pages")
flag.IntVar(&limit, "c", 0, "How many pages to get")
2016-03-07 20:04:50 +00:00
flag.Parse()
2016-03-22 20:12:37 +00:00
if len(pointLogin) < 1 || len(pointPassword) < 1 {
2016-03-21 20:56:42 +00:00
fmt.Println("Login and password must be defined")
2016-03-07 20:04:50 +00:00
return
}
if len(pointToolsToken) < 1 {
fmt.Println("Point Tools token must be defined")
return
}
2016-03-25 15:23:40 +00:00
pointClient := point.NewClient(pointApiUrl)
pointToolsClient := point_tools.NewClient(pointToolsApiUrl, pointToolsToken)
2016-03-07 20:04:50 +00:00
2016-03-22 20:12:37 +00:00
_, loginErr := pointClient.Login(pointLogin, pointPassword)
2016-03-07 20:04:50 +00:00
if loginErr != nil {
fmt.Printf("Login error %s", loginErr)
2016-03-15 22:21:50 +00:00
return
} else {
fmt.Printf("Successfully authenticated!\n")
}
2016-03-07 20:04:50 +00:00
var page point.Page
var reqErr error = nil
if 0 != fromUid {
page, reqErr = pointClient.GetNextAllPostsPageBeforeUid(fromUid)
} else {
page, reqErr = pointClient.GetRecentAllPostsPage()
}
2016-03-07 20:04:50 +00:00
if reqErr != nil {
log.Fatal(reqErr)
2016-03-15 22:21:50 +00:00
return
}
2016-03-07 20:04:50 +00:00
2016-03-15 22:21:50 +00:00
fmt.Printf("1 page requested\n")
2016-03-07 20:04:50 +00:00
if len(page.Posts) > 0 {
fmt.Println("Last uid", strconv.Itoa(page.Posts[len(page.Posts)-1].Uid))
}
sendResp, sendErr := pointToolsClient.SendPage(page)
if sendErr != nil {
log.Fatal(sendErr)
}
if point_tools.STATUS_SUCCESS != sendResp.Status {
fmt.Println("Request error", sendResp.Error.Message)
}
if false == sendResp.Data.Continue && false == forceContinue {
fmt.Println("API rejected next page request")
fmt.Println("Exiting.")
return
}
pageNumber := 1
2016-03-07 20:04:50 +00:00
2016-03-15 22:21:50 +00:00
for page.HasNext {
pageNumber++
2016-03-07 20:04:50 +00:00
2016-03-27 23:52:29 +00:00
if limit > 0 && pageNumber > limit {
fmt.Println("Limit reached")
break
}
page, reqErr = pointClient.GetNextAllPostsPage(page)
2016-03-07 20:04:50 +00:00
if reqErr != nil {
log.Fatal(reqErr)
2016-03-15 22:21:50 +00:00
return
2016-03-07 20:04:50 +00:00
}
fmt.Printf("%d page requested", pageNumber)
if len(page.Posts) > 0 {
fmt.Printf(", last uid %d", page.Posts[len(page.Posts)-1].Uid)
}
fmt.Printf(" -> %d posts\n", len(page.Posts))
2016-03-15 22:56:51 +00:00
sendResp, sendErr = pointToolsClient.SendPage(page)
2016-03-07 20:04:50 +00:00
if sendErr != nil {
log.Fatal(sendErr)
}
if point_tools.STATUS_SUCCESS != sendResp.Status {
fmt.Println("Request error", sendResp.Error.Message)
}
if false == sendResp.Data.Continue && false == forceContinue {
fmt.Println("API rejected next page request")
break
}
time.Sleep(time.Second)
2016-03-07 20:04:50 +00:00
}
fmt.Println("Exiting.")
2016-03-07 20:04:50 +00:00
}