Test point API client implementation.

point/client.go

@@ -0,0 +1,192 @@
+package point
+
+import (
+	"log"
+	"net/http"
+	"net/url"
+	"io/ioutil"
+	"encoding/json"
+	"errors"
+	"strings"
+)
+
+const point_api_url string = "https://point.im/api/"
+
+type Client struct {
+	token Token
+}
+
+func (c *Client) getResponseBody(resp *http.Response) ([]byte, error) {
+	body, read_err := ioutil.ReadAll(resp.Body)
+
+	if read_err != nil {
+		log.Fatal(read_err)
+		return nil, read_err
+	}
+
+	return body, nil
+}
+
+func (c *Client) MakeGetRequest(url string, params *url.Values) ([]byte, error) {
+	var body []byte
+
+	if params != nil {
+		url += "?" + params.Encode()
+	}
+
+	req, req_err := http.NewRequest("GET", url, nil)
+
+	if req_err != nil {
+		log.Fatal(req_err)
+		return body, req_err
+	}
+
+	if len(c.token.AuthToken) > 0 {
+		req.Header.Set("Authorization", c.token.AuthToken)
+	}
+
+	hc := &http.Client{}
+
+	resp, resp_err := hc.Do(req)
+
+	if resp_err != nil {
+		log.Fatal(resp_err)
+		return nil, resp_err
+	}
+
+	defer resp.Body.Close()
+
+	body, read_err := c.getResponseBody(resp)
+
+	if read_err != nil {
+		return nil, read_err
+	}
+
+	return body, nil
+}
+
+func (c *Client) MakePostRequest(url string, params url.Values) ([]byte, error) {
+	req, req_err := http.NewRequest("POST", url, strings.NewReader(params.Encode()))
+
+	if req_err != nil {
+		log.Fatal(req_err)
+		return nil, req_err
+	}
+
+	hc := &http.Client{}
+
+	if len(c.token.AuthToken) > 0 {
+		req.Header.Set("Authorization", c.token.AuthToken)
+	}
+
+	resp, resp_err := hc.Do(req)
+
+	if resp_err != nil {
+		log.Fatal(resp_err)
+		return nil, resp_err
+	}
+
+	body, read_err := c.getResponseBody(resp)
+
+	if read_err != nil {
+		return nil, read_err
+	}
+
+	return body, nil
+}
+
+func (c *Client) Login(login, password string) (Token, error) {
+	var token Token
+
+	data := url.Values{}
+	data.Add("login", login)
+	data.Add("password", password)
+
+	body, req_err := c.MakePostRequest(point_api_url + "login", data)
+
+	if req_err != nil {
+		return token, req_err
+	}
+
+	json_err := json.Unmarshal(body, &token)
+
+	if json_err != nil {
+		log.Fatal(json_err)
+		return token, errors.New("JSON deserialization error")
+	}
+
+	if token.Error != "" {
+		return token, errors.New(token.Error)
+	}
+
+	c.token = token
+
+	return token, nil
+}
+
+func (c *Client) GetRecentAllPostsPage() (Page, error) {
+	var page Page
+
+	if len(c.token.AuthToken) == 0 {
+		log.Fatal("Can not get recent posts. Login first.")
+		return page, errors.New("Login first")
+	}
+
+	body, req_err := c.MakeGetRequest(point_api_url + "all", nil)
+
+	if req_err != nil {
+		return page, errors.New("HTTP request error")
+	}
+
+	json_err := json.Unmarshal(body, &page)
+
+	if json_err != nil {
+		log.Fatal(json_err)
+		return page, errors.New("JSON deserialization error")
+	}
+
+	// Move page.Error to response object
+	if page.Error == "NotAuthorized" {
+		log.Fatal("Please use correct login and password to fetch recent posts")
+		return page, errors.New("Not authorized")
+	}
+
+	return page, nil
+}
+
+func (c *Client) GetNextAllPostsPage(page Page) (Page, error) {
+	var nextPage Page
+
+	if false == page.HasNext || len(page.Posts) == 0 {
+		return nextPage, errors.New("Page must have has_next=true and some posts")
+	}
+
+	if len(c.token.AuthToken) == 0 {
+		log.Fatal("Can not get recent posts. Login first.")
+		return nextPage, errors.New("Login first")
+	}
+
+	data := url.Values{}
+	data.Add("before", string(page.Posts[len(page.Posts)-1].Uid))
+
+	body, req_err := c.MakeGetRequest(point_api_url + "all", &data)
+
+	if req_err != nil {
+		return nextPage, errors.New("HTTP request error")
+	}
+
+	json_err := json.Unmarshal(body, &nextPage)
+
+	if json_err != nil {
+		log.Fatal(json_err)
+		return nextPage, errors.New("JSON deserialization error")
+	}
+
+	// Move page.Error to response object
+	if nextPage.Error == "NotAuthorized" {
+		log.Fatal("Please use correct login and password to fetch recent posts")
+		return nextPage, errors.New("Not authorized")
+	}
+
+	return nextPage, nil
+}

point/meta_post.go

@@ -0,0 +1,11 @@
+package point
+
+type MetaPost struct {
+	Bookmarked  bool           `json:"bookmarked"`
+	Uid         int            `json:"uid"`
+	Subscribed  bool           `json:"subscribed"`
+	Editable    bool           `json:"editable"`
+	Recommended bool           `json:"recommended"`
+	Rec         Recommendation `json:"rec"`
+	Post        Post           `json:"post"`
+}

point/page.go

@@ -0,0 +1,7 @@
+package point
+
+type Page struct {
+	HasNext bool         `json:"has_next"`
+	Posts   []MetaPost `json:"posts"`
+	Error   string       `json:"error"`
+}

point/post.go

@@ -0,0 +1,12 @@
+package point
+
+type Post struct {
+	Id            string     `json:"id"`
+	Author        User       `json:"author"`
+	Tags          [10]string `json:"tags"`
+	Text          string     `json:"text"`
+	CommentsCount int        `json:"comments_count"`
+	Created       string     `json:"created"`
+	Pinned        bool       `json:"pinned"`
+	Private       bool       `json:"private"`
+}

point/recommendation.go

@@ -0,0 +1,7 @@
+package point
+
+type Recommendation struct {
+	Text       string `json:"text"`
+	Comment_id int    `json:"comment_id"`
+	Author     User   `json:"author"`
+}

point/token.go

@@ -0,0 +1,7 @@
+package point
+
+type Token struct {
+	AuthToken string `json:"token"`
+	CsRfToken string `json:"csrf_token"`
+	Error     string `json:"error"`
+}

point/user.go

@@ -0,0 +1,6 @@
+package point
+
+type User struct {
+	Id    int    `json:"id"`
+	Login string `json:"login"`
+}

point_post_crawler.go

@@ -1,62 +1,19 @@
 package main
 
 import (
-	"fmt"
 	"flag"
+	"fmt"
 	//"gopkg.in/yaml.v2"
 	//"net/http"
-	"encoding/json"
+	"bitbucket.org/skobkin/point_post_crawler/point"
 	"time"
-	"net/http"
 	"log"
-	"io/ioutil"
 )
 
-type user struct {
-	id int
-	login string
-}
 
-type tag struct {
-	name string
-}
-
-type recommendation struct {
-	text string
-	comment_id int
-	author user
-}
-
-type post struct {
-	pinned bool
-	tags [10]int
-	comments_count int
-	author user
-	text string
-	created string
-	id string
-	private bool
-}
-
-type meta_post struct {
-	bookmarked bool
-	uid int
-	subscribed bool
-	editable bool
-	recommended bool
-	rec recommendation
-	post post
-}
-
-type page struct {
-	has_next bool
-	posts [20]meta_post
-}
-
-const point_api_url string = "https://point.im/api/"
 
 func main() {
-	var login, password string;
+	var login, password string
 
 	flag.StringVar(&login, "l", "", "Account login")
 	flag.StringVar(&password, "p", "", "Account password")
@@ -67,43 +24,43 @@ func main() {
 		return
 	}
 
-	// Auth
+	client := point.Client{}
 
+	_, login_err := client.Login(login, password)
 
-	// Post processing
-	page_number := 0;
-	for {
-		resp, req_err := http.Get(point_api_url + "recent")
+	if login_err != nil {
+		fmt.Printf("Login error %s", login_err)
+		return
+	} else {
+		fmt.Printf("Successfully authenticated!\n")
+	}
+
+	page, req_err := client.GetRecentAllPostsPage()
 
 	if req_err != nil {
 		log.Fatal(req_err)
-			continue
+		return
 	}
 
-		defer resp.Body.Close()
+	fmt.Printf("1 page requested\n")
+	fmt.Printf("%d posts\n", len(page.Posts))
 
-		body, read_err := ioutil.ReadAll(resp.Body)
+	pageNumber := 1;
 
-		if read_err != nil {
-			log.Fatal(read_err)
-			continue
+	for page.HasNext {
+		pageNumber++
+
+		page, req_err := client.GetNextAllPostsPage(page)
+
+		if req_err != nil {
+			log.Fatal(req_err)
+			return
 		}
 
-		var page page
+		fmt.Printf("%d page requested\n", pageNumber)
+		fmt.Printf("%d posts\n", len(page.Posts))
 
-		json_err := json.Unmarshal(body, &page)
-
-		if json_err != nil {
-			log.Fatal(json_err)
-			continue
-		}
-
-		page_number++
-		fmt.Printf("%d done:\n", page_number)
-
-		//fmt.Printf("%+v", body)
-		fmt.Printf("%s\n", body)
-
-		time.Sleep(10 * time.Second)
+		time.Sleep(2 * time.Second)
 	}
 }
+