point-tools-crawler/point_post_crawler.go

package main

import (
	"flag"
	"fmt"
	"bitbucket.org/skobkin/point-tools-crawler/point"
	"bitbucket.org/skobkin/point-tools-crawler/point_tools"
	"log"
	"time"
)

func main() {
	var login, password string

	flag.StringVar(&login, "l", "", "Account login")
	flag.StringVar(&password, "p", "", "Account password")
	flag.Parse()

	if len(login) < 1 || len(password) < 1 {
		fmt.Printf("Login and password must be defined")
		return
	}

	pointClient := point.NewClient("https://point.im/api/")
	pointToolsClient := point_tools.NewClient("http://point-tools.local:8000/api/crawler/")

	_, login_err := pointClient.Login(login, password)

	if login_err != nil {
		fmt.Printf("Login error %s", login_err)
		return
	} else {
		fmt.Printf("Successfully authenticated!\n")
	}

	page, req_err := pointClient.GetRecentAllPostsPage()

	if req_err != nil {
		log.Fatal(req_err)
		return
	}

	fmt.Printf("1 page requested\n")

	if len(page.Posts) == 0 {
		fmt.Println("Last uid", string(page.Posts[len(page.Posts)-1].Uid))
	}

	pointToolsClient.SendPage(page)

	//var str string

	pageNumber := 1

	for page.HasNext {
		pageNumber++

		page, req_err = pointClient.GetNextAllPostsPage(page)

		if req_err != nil {
			log.Fatal(req_err)
			return
		}



		fmt.Printf("%d page requested, last uid: %d", pageNumber)
		fmt.Printf("%d posts\n", len(page.Posts))

		pointToolsClient.SendPage(page)

		time.Sleep(time.Second)
	}
}