point-tools-crawler/point_post_crawler.go

package main

import (
	"bitbucket.org/skobkin/point-api-go"
	"bitbucket.org/skobkin/point-tools-go"
	"gopkg.in/alecthomas/kingpin.v2"
	"fmt"
	"log"
	"strconv"
	"time"
)

var (
	pointApiUrl = kingpin.Flag("point-api", "Point.im API URL").Default("https://point.im/api/").String()
	pointToolsApiUrl = kingpin.Flag("point-tools-api", "Point Tools API URL").Default("https://point.skobk.in/api/crawler/").String()
	pointLogin = kingpin.Flag("login", "Point.im login").Short('l').Required().String()
	pointPassword = kingpin.Flag("password", "Point.im password").Short('p').Required().String()
	pointToolsToken = kingpin.Flag("point-tools-token", "Point Tools secret token for crawler").Short('t').Required().String()
	forceContinue = kingpin.Flag("force-continue", "Keep going even after API rejected new page").Short('f').Bool()
	fromUid = kingpin.Flag("from-uid", "Start after provided post UID").Short('u').Default("0").Int()
	limit = kingpin.Flag("limit", "How many pages to get").Short('c').Default("0").Int()
)

func main() {
	kingpin.Parse()

	pointClient := point.New(*pointApiUrl)
	pointToolsClient := point_tools.New(*pointToolsApiUrl, *pointToolsToken)

	_, loginErr := pointClient.Login(*pointLogin, *pointPassword)

	if loginErr != nil {
		fmt.Printf("Login error %s", loginErr)
		return
	} else {
		fmt.Printf("Successfully authenticated!\n")
	}

	var page point.Page
	var reqErr error = nil

	if 0 != *fromUid {
		page, reqErr = pointClient.GetNextAllPostsPageBeforeUid(*fromUid)
	} else {
		page, reqErr = pointClient.GetRecentAllPostsPage()
	}

	if reqErr != nil {
		log.Fatal(reqErr)
		return
	}

	fmt.Printf("1 page requested\n")

	if len(page.Posts) > 0 {
		fmt.Println("Last uid", strconv.Itoa(page.Posts[len(page.Posts)-1].Uid))
	}

	sendResp, sendErr := pointToolsClient.SendAllPage(page)

	if sendErr != nil {
		log.Fatal(sendErr)
	}

	if point_tools.STATUS_SUCCESS != sendResp.Status {
		fmt.Println("Request error", sendResp.Error.Message)
	}

	if false == sendResp.Data.Continue && false == *forceContinue {
		fmt.Println("API rejected next page request")
		fmt.Println("Exiting.")
		return
	}

	pageNumber := 1

	for page.HasNext {
		pageNumber++

		if *limit > 0 && pageNumber > *limit {
			fmt.Println("Limit reached")
			break
		}

		page, reqErr = pointClient.GetNextAllPostsPage(page)

		if reqErr != nil {
			log.Fatal(reqErr)
			return
		}

		fmt.Printf("%d page requested", pageNumber)
		if len(page.Posts) > 0 {
			fmt.Printf(", last uid %d", page.Posts[len(page.Posts)-1].Uid)
		}
		fmt.Printf(" -> %d posts\n", len(page.Posts))

		sendResp, sendErr = pointToolsClient.SendAllPage(page)

		if sendErr != nil {
			log.Fatal(sendErr)
		}

		if point_tools.STATUS_SUCCESS != sendResp.Status {
			fmt.Println("Request error", sendResp.Error.Message)
		}

		if false == sendResp.Data.Continue && false == *forceContinue {
			fmt.Println("API rejected next page request")
			break
		}

		time.Sleep(time.Second)
	}

	fmt.Println("Exiting.")
}