118 lines
2.9 KiB
Go
118 lines
2.9 KiB
Go
package main
|
|
|
|
import (
|
|
"bitbucket.org/skobkin/point-api-go"
|
|
"bitbucket.org/skobkin/point-tools-go"
|
|
"gopkg.in/alecthomas/kingpin.v2"
|
|
"fmt"
|
|
"log"
|
|
"strconv"
|
|
"time"
|
|
)
|
|
|
|
var (
|
|
pointApiUrl = kingpin.Flag("point-api", "Point.im API URL").Default("https://point.im/api/").String()
|
|
pointToolsApiUrl = kingpin.Flag("point-tools-api", "Point Tools API URL").Default("https://point.skobk.in/api/crawler/").String()
|
|
pointLogin = kingpin.Flag("login", "Point.im login").Short('l').Required().String()
|
|
pointPassword = kingpin.Flag("password", "Point.im password").Short('p').Required().String()
|
|
pointToolsToken = kingpin.Flag("point-tools-token", "Point Tools secret token for crawler").Short('t').Required().String()
|
|
forceContinue = kingpin.Flag("force-continue", "Keep going even after API rejected new page").Short('f').Bool()
|
|
fromUid = kingpin.Flag("from-uid", "Start after provided post UID").Short('u').Default("0").Int()
|
|
limit = kingpin.Flag("limit", "How many pages to get").Short('c').Default("0").Int()
|
|
)
|
|
|
|
func main() {
|
|
kingpin.Parse()
|
|
|
|
pointClient := point.New(*pointApiUrl)
|
|
pointToolsClient := point_tools.New(*pointToolsApiUrl, *pointToolsToken)
|
|
|
|
_, loginErr := pointClient.Login(*pointLogin, *pointPassword)
|
|
|
|
if loginErr != nil {
|
|
fmt.Printf("Login error %s", loginErr)
|
|
return
|
|
} else {
|
|
fmt.Printf("Successfully authenticated!\n")
|
|
}
|
|
|
|
var page point.Page
|
|
var reqErr error = nil
|
|
|
|
if 0 != *fromUid {
|
|
page, reqErr = pointClient.GetNextAllPostsPageBeforeUid(*fromUid)
|
|
} else {
|
|
page, reqErr = pointClient.GetRecentAllPostsPage()
|
|
}
|
|
|
|
if reqErr != nil {
|
|
log.Fatal(reqErr)
|
|
return
|
|
}
|
|
|
|
fmt.Printf("1 page requested\n")
|
|
|
|
if len(page.Posts) > 0 {
|
|
fmt.Println("Last uid", strconv.Itoa(page.Posts[len(page.Posts)-1].Uid))
|
|
}
|
|
|
|
sendResp, sendErr := pointToolsClient.SendAllPage(page)
|
|
|
|
if sendErr != nil {
|
|
log.Fatal(sendErr)
|
|
}
|
|
|
|
if point_tools.STATUS_SUCCESS != sendResp.Status {
|
|
fmt.Println("Request error", sendResp.Error.Message)
|
|
}
|
|
|
|
if false == sendResp.Data.Continue && false == *forceContinue {
|
|
fmt.Println("API rejected next page request")
|
|
fmt.Println("Exiting.")
|
|
return
|
|
}
|
|
|
|
pageNumber := 1
|
|
|
|
for page.HasNext {
|
|
pageNumber++
|
|
|
|
if *limit > 0 && pageNumber > *limit {
|
|
fmt.Println("Limit reached")
|
|
break
|
|
}
|
|
|
|
page, reqErr = pointClient.GetNextAllPostsPage(page)
|
|
|
|
if reqErr != nil {
|
|
log.Fatal(reqErr)
|
|
return
|
|
}
|
|
|
|
fmt.Printf("%d page requested", pageNumber)
|
|
if len(page.Posts) > 0 {
|
|
fmt.Printf(", last uid %d", page.Posts[len(page.Posts)-1].Uid)
|
|
}
|
|
fmt.Printf(" -> %d posts\n", len(page.Posts))
|
|
|
|
sendResp, sendErr = pointToolsClient.SendAllPage(page)
|
|
|
|
if sendErr != nil {
|
|
log.Fatal(sendErr)
|
|
}
|
|
|
|
if point_tools.STATUS_SUCCESS != sendResp.Status {
|
|
fmt.Println("Request error", sendResp.Error.Message)
|
|
}
|
|
|
|
if false == sendResp.Data.Continue && false == *forceContinue {
|
|
fmt.Println("API rejected next page request")
|
|
break
|
|
}
|
|
|
|
time.Sleep(time.Second)
|
|
}
|
|
|
|
fmt.Println("Exiting.")
|
|
}
|