mirror of
https://bitbucket.org/skobkin/point-tools-crawler.git
synced 2024-12-04 18:25:52 +00:00
CLI options refactored to the new library.
This commit is contained in:
parent
8c5c4cf430
commit
6f412ee8c3
|
@ -3,44 +3,31 @@ package main
|
||||||
import (
|
import (
|
||||||
"bitbucket.org/skobkin/point-api-go"
|
"bitbucket.org/skobkin/point-api-go"
|
||||||
"bitbucket.org/skobkin/point-tools-go"
|
"bitbucket.org/skobkin/point-tools-go"
|
||||||
"flag"
|
"gopkg.in/alecthomas/kingpin.v2"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
"strconv"
|
"strconv"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
pointApiUrl = kingpin.Flag("point-api", "Point.im API URL").Default("https://point.im/api/").String()
|
||||||
|
pointToolsApiUrl = kingpin.Flag("point-tools-api", "Point Tools API URL").Default("https://point.skobk.in/api/crawler/").String()
|
||||||
|
pointLogin = kingpin.Flag("login", "Point.im login").Short('l').Required().String()
|
||||||
|
pointPassword = kingpin.Flag("password", "Point.im password").Short('p').Required().String()
|
||||||
|
pointToolsToken = kingpin.Flag("point-tools-token", "Point Tools secret token for crawler").Short('t').Required().String()
|
||||||
|
forceContinue = kingpin.Flag("force-continue", "Keep going even after API rejected new page").Short('f').Bool()
|
||||||
|
fromUid = kingpin.Flag("from-uid", "Start after provided post UID").Short('u').Default("0").Int()
|
||||||
|
limit = kingpin.Flag("limit", "How many pages to get").Short('c').Default("0").Int()
|
||||||
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
var pointApiUrl, pointToolsApiUrl string;
|
kingpin.Parse()
|
||||||
var pointLogin, pointPassword, pointToolsToken string
|
|
||||||
var forceContinue bool
|
|
||||||
var fromUid, limit int
|
|
||||||
|
|
||||||
// Todo refactor the CLI options to new library
|
pointClient := point.New(*pointApiUrl)
|
||||||
flag.StringVar(&pointLogin, "l", "", "Account login")
|
pointToolsClient := point_tools.New(*pointToolsApiUrl, *pointToolsToken)
|
||||||
flag.StringVar(&pointPassword, "p", "", "Account password")
|
|
||||||
flag.StringVar(&pointToolsToken, "t", "", "Point Tools crawler API token")
|
|
||||||
flag.StringVar(&pointApiUrl, "s", "https://point.im/api/", "Point.im API url")
|
|
||||||
flag.StringVar(&pointToolsApiUrl, "g", "https://point.skobk.in/api/crawler/", "Point Tools API url")
|
|
||||||
flag.BoolVar(&forceContinue, "f", false, "Force continue reading /all despite of server refusal")
|
|
||||||
flag.IntVar(&fromUid, "u", 0, "From which UID start to load pages")
|
|
||||||
flag.IntVar(&limit, "c", 0, "How many pages to get")
|
|
||||||
flag.Parse()
|
|
||||||
|
|
||||||
if len(pointLogin) < 1 || len(pointPassword) < 1 {
|
_, loginErr := pointClient.Login(*pointLogin, *pointPassword)
|
||||||
fmt.Println("Login and password must be defined")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(pointToolsToken) < 1 {
|
|
||||||
fmt.Println("Point Tools token must be defined")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
pointClient := point.New(pointApiUrl)
|
|
||||||
pointToolsClient := point_tools.New(pointToolsApiUrl, pointToolsToken)
|
|
||||||
|
|
||||||
_, loginErr := pointClient.Login(pointLogin, pointPassword)
|
|
||||||
|
|
||||||
if loginErr != nil {
|
if loginErr != nil {
|
||||||
fmt.Printf("Login error %s", loginErr)
|
fmt.Printf("Login error %s", loginErr)
|
||||||
|
@ -52,8 +39,8 @@ func main() {
|
||||||
var page point.Page
|
var page point.Page
|
||||||
var reqErr error = nil
|
var reqErr error = nil
|
||||||
|
|
||||||
if 0 != fromUid {
|
if 0 != *fromUid {
|
||||||
page, reqErr = pointClient.GetNextAllPostsPageBeforeUid(fromUid)
|
page, reqErr = pointClient.GetNextAllPostsPageBeforeUid(*fromUid)
|
||||||
} else {
|
} else {
|
||||||
page, reqErr = pointClient.GetRecentAllPostsPage()
|
page, reqErr = pointClient.GetRecentAllPostsPage()
|
||||||
}
|
}
|
||||||
|
@ -79,7 +66,7 @@ func main() {
|
||||||
fmt.Println("Request error", sendResp.Error.Message)
|
fmt.Println("Request error", sendResp.Error.Message)
|
||||||
}
|
}
|
||||||
|
|
||||||
if false == sendResp.Data.Continue && false == forceContinue {
|
if false == sendResp.Data.Continue && false == *forceContinue {
|
||||||
fmt.Println("API rejected next page request")
|
fmt.Println("API rejected next page request")
|
||||||
fmt.Println("Exiting.")
|
fmt.Println("Exiting.")
|
||||||
return
|
return
|
||||||
|
@ -90,7 +77,7 @@ func main() {
|
||||||
for page.HasNext {
|
for page.HasNext {
|
||||||
pageNumber++
|
pageNumber++
|
||||||
|
|
||||||
if limit > 0 && pageNumber > limit {
|
if *limit > 0 && pageNumber > *limit {
|
||||||
fmt.Println("Limit reached")
|
fmt.Println("Limit reached")
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
@ -118,7 +105,7 @@ func main() {
|
||||||
fmt.Println("Request error", sendResp.Error.Message)
|
fmt.Println("Request error", sendResp.Error.Message)
|
||||||
}
|
}
|
||||||
|
|
||||||
if false == sendResp.Data.Continue && false == forceContinue {
|
if false == sendResp.Data.Continue && false == *forceContinue {
|
||||||
fmt.Println("API rejected next page request")
|
fmt.Println("API rejected next page request")
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue