131 lines
3.0 KiB
Go
131 lines
3.0 KiB
Go
package main
|
|
|
|
import (
|
|
"bitbucket.org/skobkin/point-api-go"
|
|
"bitbucket.org/skobkin/point-tools-go"
|
|
"flag"
|
|
"fmt"
|
|
"log"
|
|
"strconv"
|
|
"time"
|
|
)
|
|
|
|
func main() {
|
|
var pointApiUrl, pointToolsApiUrl string;
|
|
var pointLogin, pointPassword, pointToolsToken string
|
|
var forceContinue bool
|
|
var fromUid, limit int
|
|
|
|
// Todo refactor the CLI options to new library
|
|
flag.StringVar(&pointLogin, "l", "", "Account login")
|
|
flag.StringVar(&pointPassword, "p", "", "Account password")
|
|
flag.StringVar(&pointToolsToken, "t", "", "Point Tools crawler API token")
|
|
flag.StringVar(&pointApiUrl, "s", "https://point.im/api/", "Point.im API url")
|
|
flag.StringVar(&pointToolsApiUrl, "g", "https://point.skobk.in/api/crawler/", "Point Tools API url")
|
|
flag.BoolVar(&forceContinue, "f", false, "Force continue reading /all despite of server refusal")
|
|
flag.IntVar(&fromUid, "u", 0, "From which UID start to load pages")
|
|
flag.IntVar(&limit, "c", 0, "How many pages to get")
|
|
flag.Parse()
|
|
|
|
if len(pointLogin) < 1 || len(pointPassword) < 1 {
|
|
fmt.Println("Login and password must be defined")
|
|
return
|
|
}
|
|
|
|
if len(pointToolsToken) < 1 {
|
|
fmt.Println("Point Tools token must be defined")
|
|
return
|
|
}
|
|
|
|
pointClient := point.New(pointApiUrl)
|
|
pointToolsClient := point_tools.New(pointToolsApiUrl, pointToolsToken)
|
|
|
|
_, loginErr := pointClient.Login(pointLogin, pointPassword)
|
|
|
|
if loginErr != nil {
|
|
fmt.Printf("Login error %s", loginErr)
|
|
return
|
|
} else {
|
|
fmt.Printf("Successfully authenticated!\n")
|
|
}
|
|
|
|
var page point.Page
|
|
var reqErr error = nil
|
|
|
|
if 0 != fromUid {
|
|
page, reqErr = pointClient.GetNextAllPostsPageBeforeUid(fromUid)
|
|
} else {
|
|
page, reqErr = pointClient.GetRecentAllPostsPage()
|
|
}
|
|
|
|
if reqErr != nil {
|
|
log.Fatal(reqErr)
|
|
return
|
|
}
|
|
|
|
fmt.Printf("1 page requested\n")
|
|
|
|
if len(page.Posts) > 0 {
|
|
fmt.Println("Last uid", strconv.Itoa(page.Posts[len(page.Posts)-1].Uid))
|
|
}
|
|
|
|
sendResp, sendErr := pointToolsClient.SendPage(page)
|
|
|
|
if sendErr != nil {
|
|
log.Fatal(sendErr)
|
|
}
|
|
|
|
if point_tools.STATUS_SUCCESS != sendResp.Status {
|
|
fmt.Println("Request error", sendResp.Error.Message)
|
|
}
|
|
|
|
if false == sendResp.Data.Continue && false == forceContinue {
|
|
fmt.Println("API rejected next page request")
|
|
fmt.Println("Exiting.")
|
|
return
|
|
}
|
|
|
|
pageNumber := 1
|
|
|
|
for page.HasNext {
|
|
pageNumber++
|
|
|
|
if limit > 0 && pageNumber > limit {
|
|
fmt.Println("Limit reached")
|
|
break
|
|
}
|
|
|
|
page, reqErr = pointClient.GetNextAllPostsPage(page)
|
|
|
|
if reqErr != nil {
|
|
log.Fatal(reqErr)
|
|
return
|
|
}
|
|
|
|
fmt.Printf("%d page requested", pageNumber)
|
|
if len(page.Posts) > 0 {
|
|
fmt.Printf(", last uid %d", page.Posts[len(page.Posts)-1].Uid)
|
|
}
|
|
fmt.Printf(" -> %d posts\n", len(page.Posts))
|
|
|
|
sendResp, sendErr = pointToolsClient.SendPage(page)
|
|
|
|
if sendErr != nil {
|
|
log.Fatal(sendErr)
|
|
}
|
|
|
|
if point_tools.STATUS_SUCCESS != sendResp.Status {
|
|
fmt.Println("Request error", sendResp.Error.Message)
|
|
}
|
|
|
|
if false == sendResp.Data.Continue && false == forceContinue {
|
|
fmt.Println("API rejected next page request")
|
|
break
|
|
}
|
|
|
|
time.Sleep(time.Second)
|
|
}
|
|
|
|
fmt.Println("Exiting.")
|
|
}
|