point-tools-crawler/point_post_crawler.go

package main

import (
	"flag"
	"fmt"
	"bitbucket.org/skobkin/point-tools-crawler/point"
	"bitbucket.org/skobkin/point-tools-crawler/point_tools"
	"log"
	"time"
	"strconv"
)

func main() {
	var login, password string

	flag.StringVar(&login, "l", "", "Account login")
	flag.StringVar(&password, "p", "", "Account password")
	flag.Parse()

	if len(login) < 1 || len(password) < 1 {
		fmt.Printf("Login and password must be defined")
		return
	}

	pointClient := point.NewClient("https://point.im/api/")
	pointToolsClient := point_tools.NewClient("http://point-tools.local:8000/api/crawler/")

	_, loginErr := pointClient.Login(login, password)

	if loginErr != nil {
		fmt.Printf("Login error %s", loginErr)
		return
	} else {
		fmt.Printf("Successfully authenticated!\n")
	}

	page, reqErr := pointClient.GetRecentAllPostsPage()

	if reqErr != nil {
		log.Fatal(reqErr)
		return
	}

	fmt.Printf("1 page requested\n")

	if len(page.Posts) > 0 {
		fmt.Println("Last uid", strconv.Itoa(page.Posts[len(page.Posts)-1].Uid))
	}

	sendResp, sendErr := pointToolsClient.SendPage(page)

	if sendErr != nil {
		log.Fatal(sendErr)
	}

	if point_tools.STATUS_SUCCESS != sendResp.Status || false == sendResp.Data.Continue {
		fmt.Println("API rejected next page request")
		fmt.Println("Exiting.")
		return
	}

	pageNumber := 1

	for page.HasNext {
		pageNumber++

		page, reqErr = pointClient.GetNextAllPostsPage(page)

		if reqErr != nil {
			log.Fatal(reqErr)
			return
		}

		fmt.Printf("%d page requested", pageNumber)
		if len(page.Posts) > 0 {
			fmt.Printf(", last uid %d", page.Posts[len(page.Posts)-1].Uid)
		}
		fmt.Printf(" -> %d posts\n", len(page.Posts))

		sendResp, sendErr := pointToolsClient.SendPage(page)

		if sendErr != nil {
			log.Fatal(sendErr)
		}

		if point_tools.STATUS_SUCCESS != sendResp.Status || false == sendResp.Data.Continue {
			fmt.Println("API rejected next page request")
			break
		}

		time.Sleep(time.Second)
	}

	fmt.Println("Exiting.")
}