skobkin/point-tools-crawler
1
0
Fork 0
mirror of https://bitbucket.org/skobkin/point-tools-crawler.git synced 2024-12-05 02:25:53 +00:00
Code Issues Projects Releases Wiki Activity

CLI options refactored to the new library.

Browse source
This commit is contained in:
Alexey Skobkin 2016-03-29 11:22:44 +03:00
parent 8c5c4cf430
commit 6f412ee8c3
1 changed files with 21 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

55
point_post_crawler.go
View file

@ -3,44 +3,31 @@ package main
import (
"bitbucket.org/skobkin/point-api-go"
"bitbucket.org/skobkin/point-tools-go"
"flag"
"gopkg.in/alecthomas/kingpin.v2"
"fmt"
"log"
"strconv"
"time"
)
var (
pointApiUrl = kingpin.Flag("point-api", "Point.im API URL").Default("https://point.im/api/").String()
pointToolsApiUrl = kingpin.Flag("point-tools-api", "Point Tools API URL").Default("https://point.skobk.in/api/crawler/").String()
pointLogin = kingpin.Flag("login", "Point.im login").Short('l').Required().String()
pointPassword = kingpin.Flag("password", "Point.im password").Short('p').Required().String()
pointToolsToken = kingpin.Flag("point-tools-token", "Point Tools secret token for crawler").Short('t').Required().String()
forceContinue = kingpin.Flag("force-continue", "Keep going even after API rejected new page").Short('f').Bool()
fromUid = kingpin.Flag("from-uid", "Start after provided post UID").Short('u').Default("0").Int()
limit = kingpin.Flag("limit", "How many pages to get").Short('c').Default("0").Int()
)
func main() {
var pointApiUrl, pointToolsApiUrl string;
var pointLogin, pointPassword, pointToolsToken string
var forceContinue bool
var fromUid, limit int
kingpin.Parse()
// Todo refactor the CLI options to new library
flag.StringVar(&pointLogin, "l", "", "Account login")
flag.StringVar(&pointPassword, "p", "", "Account password")
flag.StringVar(&pointToolsToken, "t", "", "Point Tools crawler API token")
flag.StringVar(&pointApiUrl, "s", "https://point.im/api/", "Point.im API url")
flag.StringVar(&pointToolsApiUrl, "g", "https://point.skobk.in/api/crawler/", "Point Tools API url")
flag.BoolVar(&forceContinue, "f", false, "Force continue reading /all despite of server refusal")
flag.IntVar(&fromUid, "u", 0, "From which UID start to load pages")
flag.IntVar(&limit, "c", 0, "How many pages to get")
flag.Parse()
pointClient := point.New(*pointApiUrl)
pointToolsClient := point_tools.New(*pointToolsApiUrl, *pointToolsToken)
if len(pointLogin) < 1 || len(pointPassword) < 1 {
fmt.Println("Login and password must be defined")
return
}
if len(pointToolsToken) < 1 {
fmt.Println("Point Tools token must be defined")
return
}
pointClient := point.New(pointApiUrl)
pointToolsClient := point_tools.New(pointToolsApiUrl, pointToolsToken)
_, loginErr := pointClient.Login(pointLogin, pointPassword)
_, loginErr := pointClient.Login(*pointLogin, *pointPassword)
if loginErr != nil {
fmt.Printf("Login error %s", loginErr)
@ -52,8 +39,8 @@ func main() {
var page point.Page
var reqErr error = nil
if 0 != fromUid {
page, reqErr = pointClient.GetNextAllPostsPageBeforeUid(fromUid)
if 0 != *fromUid {
page, reqErr = pointClient.GetNextAllPostsPageBeforeUid(*fromUid)
} else {
page, reqErr = pointClient.GetRecentAllPostsPage()
}
@ -79,7 +66,7 @@ func main() {
fmt.Println("Request error", sendResp.Error.Message)
}
if false == sendResp.Data.Continue && false == forceContinue {
if false == sendResp.Data.Continue && false == *forceContinue {
fmt.Println("API rejected next page request")
fmt.Println("Exiting.")
return
@ -90,7 +77,7 @@ func main() {
for page.HasNext {
pageNumber++
if limit > 0 && pageNumber > limit {
if *limit > 0 && pageNumber > *limit {
fmt.Println("Limit reached")
break
}
@ -118,7 +105,7 @@ func main() {
fmt.Println("Request error", sendResp.Error.Message)
}
if false == sendResp.Data.Continue && false == forceContinue {
if false == sendResp.Data.Continue && false == *forceContinue {
fmt.Println("API rejected next page request")
break
}