point-tools-crawler/point_post_crawler.go

package main

import (
	"flag"
	"fmt"
	//"gopkg.in/yaml.v2"
	//"net/http"
	"bitbucket.org/skobkin/point-tools-crawler/point"
	"time"
	"log"
	"encoding/json"
	"net/url"
)



func main() {
	var login, password string

	flag.StringVar(&login, "l", "", "Account login")
	flag.StringVar(&password, "p", "", "Account password")
	flag.Parse()

	if len(login) < 1 || len(password) < 1 {
		fmt.Printf("Login and password must be defined")
		return
	}

	client := point.Client{}

	_, login_err := client.Login(login, password)

	if login_err != nil {
		fmt.Printf("Login error %s", login_err)
		return
	} else {
		fmt.Printf("Successfully authenticated!\n")
	}

	page, req_err := client.GetRecentAllPostsPage()

	if req_err != nil {
		log.Fatal(req_err)
		return
	}

	fmt.Printf("1 page requested\n")
	printPageJSON(page)
	sendPage(page)

	pageNumber := 1;

	for page.HasNext {
		pageNumber++

		page, req_err = client.GetNextAllPostsPage(page)

		if req_err != nil {
			log.Fatal(req_err)
			return
		}

		printPageJSON(page)
		sendPage(page)

		fmt.Printf("%d page requested\n", pageNumber)
		fmt.Printf("%d posts\n", len(page.Posts))

		time.Sleep(time.Second)
	}
}

func getPageJSON(page point.Page) (string, error) {
	b, err := json.Marshal(page)

	if err != nil {
		fmt.Println("error:", err)
		return "", err
	}

	return string(b[:]), nil
}

func printPageJSON(page point.Page) {
	json, err := getPageJSON(page)

	if err != nil {
		fmt.Println("JSON print error")
	}

	fmt.Println(json)
}

func sendPage(page point.Page) error {
	json, err := getPageJSON(page)

	if err != nil {
		return err
	}

	client := point.Client{}

	data := url.Values{}
	data.Set("token", "test_token")
	data.Add("json", json)

	body, req_err := client.MakePostRequest("http://point-tools.local:8000/api/crawler/all/page", data)

	if req_err != nil {
		return req_err
	}

	fmt.Println("Response:", string(body[:]))

	return nil
}