point-tools-crawler/point_post_crawler.go

package main

import (
	"bitbucket.org/skobkin/point-tools-crawler/point"
	"bitbucket.org/skobkin/point-tools-crawler/point_tools"
	"flag"
	"fmt"
	"log"
	"strconv"
	"time"
)

func main() {
	var pointApiUrl, pointToolsApiUrl string;
	var pointLogin, pointPassword, pointToolsToken string
	var forceContinue bool
	var fromUid, limit int

	// Todo refactor the CLI options to new library
	flag.StringVar(&pointLogin, "l", "", "Account login")
	flag.StringVar(&pointPassword, "p", "", "Account password")
	flag.StringVar(&pointToolsToken, "t", "", "Point Tools crawler API token")
	flag.StringVar(&pointApiUrl, "s", "https://point.im/api/", "Point.im API url")
	flag.StringVar(&pointToolsApiUrl, "g", "https://point.skobk.in/api/crawler/", "Point Tools API url")
	flag.BoolVar(&forceContinue, "f", false, "Force continue reading /all despite of server refusal")
	flag.IntVar(&fromUid, "u", 0, "From which UID start to load pages")
	flag.IntVar(&limit, "c", 0, "How many pages to get")
	flag.Parse()

	if len(pointLogin) < 1 || len(pointPassword) < 1 {
		fmt.Println("Login and password must be defined")
		return
	}

	if len(pointToolsToken) < 1 {
		fmt.Println("Point Tools token must be defined")
		return
	}

	pointClient := point.NewClient(pointApiUrl)
	pointToolsClient := point_tools.NewClient(pointToolsApiUrl, pointToolsToken)

	_, loginErr := pointClient.Login(pointLogin, pointPassword)

	if loginErr != nil {
		fmt.Printf("Login error %s", loginErr)
		return
	} else {
		fmt.Printf("Successfully authenticated!\n")
	}

	var page point.Page
	var reqErr error = nil

	if 0 != fromUid {
		page, reqErr = pointClient.GetNextAllPostsPageBeforeUid(fromUid)
	} else {
		page, reqErr = pointClient.GetRecentAllPostsPage()
	}

	if reqErr != nil {
		log.Fatal(reqErr)
		return
	}

	fmt.Printf("1 page requested\n")

	if len(page.Posts) > 0 {
		fmt.Println("Last uid", strconv.Itoa(page.Posts[len(page.Posts)-1].Uid))
	}

	sendResp, sendErr := pointToolsClient.SendPage(page)

	if sendErr != nil {
		log.Fatal(sendErr)
	}

	if point_tools.STATUS_SUCCESS != sendResp.Status {
		fmt.Println("Request error", sendResp.Error.Message)
	}

	if false == sendResp.Data.Continue && false == forceContinue {
		fmt.Println("API rejected next page request")
		fmt.Println("Exiting.")
		return
	}

	pageNumber := 1

	for page.HasNext {
		pageNumber++

		if limit > 0 && pageNumber > limit {
			fmt.Println("Limit reached")
			break
		}

		page, reqErr = pointClient.GetNextAllPostsPage(page)

		if reqErr != nil {
			log.Fatal(reqErr)
			return
		}

		fmt.Printf("%d page requested", pageNumber)
		if len(page.Posts) > 0 {
			fmt.Printf(", last uid %d", page.Posts[len(page.Posts)-1].Uid)
		}
		fmt.Printf(" -> %d posts\n", len(page.Posts))

		sendResp, sendErr = pointToolsClient.SendPage(page)

		if sendErr != nil {
			log.Fatal(sendErr)
		}

		if point_tools.STATUS_SUCCESS != sendResp.Status {
			fmt.Println("Request error", sendResp.Error.Message)
		}

		if false == sendResp.Data.Continue && false == forceContinue {
			fmt.Println("API rejected next page request")
			break
		}

		time.Sleep(time.Second)
	}

	fmt.Println("Exiting.")
}