HTTP client functionality refactored. PointToolsClient added.

point/client.go

@@ -1,112 +1,37 @@
 package point
 
 import (
+	"bitbucket.org/skobkin/point-tools-crawler/simple_http"
+	"encoding/json"
+	"errors"
 	"log"
 	"net/http"
 	"net/url"
-	"io/ioutil"
-	"encoding/json"
-	"errors"
-	"strings"
 	"strconv"
 )
 
-const point_api_url string = "https://point.im/api/"
+type PointClient struct {
-
+	client simple_http.Client
-type Client struct {
+	apiUrl string
-	token Token
+	token  Token
 }
 
-func (c *Client) getResponseBody(resp *http.Response) ([]byte, error) {
+func NewClient(apiUrl string) PointClient {
-	body, read_err := ioutil.ReadAll(resp.Body)
+	return PointClient{
-
+		simple_http.Client{http.Client{}},
-	if read_err != nil {
+		apiUrl,
-		log.Fatal(read_err)
+		Token{},
-		return nil, read_err
 	}
-
-	return body, nil
 }
 
-func (c *Client) MakeGetRequest(url string, params *url.Values) ([]byte, error) {
+func (c *PointClient) Login(login, password string) (Token, error) {
-	var body []byte
-
-	if params != nil {
-		url += "?" + params.Encode()
-	}
-
-	req, req_err := http.NewRequest("GET", url, nil)
-
-	if req_err != nil {
-		log.Fatal(req_err)
-		return body, req_err
-	}
-
-	if len(c.token.AuthToken) > 0 {
-		req.Header.Set("Authorization", c.token.AuthToken)
-	}
-
-	hc := &http.Client{}
-
-	resp, resp_err := hc.Do(req)
-
-	if resp_err != nil {
-		log.Fatal(resp_err)
-		return nil, resp_err
-	}
-
-	defer resp.Body.Close()
-
-	body, read_err := c.getResponseBody(resp)
-
-	if read_err != nil {
-		return nil, read_err
-	}
-
-	return body, nil
-}
-
-func (c *Client) MakePostRequest(url string, params url.Values) ([]byte, error) {
-	req, req_err := http.NewRequest("POST", url, strings.NewReader(params.Encode()))
-
-	if req_err != nil {
-		log.Fatal(req_err)
-		return nil, req_err
-	}
-
-	hc := &http.Client{}
-
-	if len(c.token.AuthToken) > 0 {
-		req.Header.Set("Authorization", c.token.AuthToken)
-	}
-
-	// Adding necessary for POST request header
-	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
-
-	resp, resp_err := hc.Do(req)
-
-	if resp_err != nil {
-		log.Fatal(resp_err)
-		return nil, resp_err
-	}
-
-	body, read_err := c.getResponseBody(resp)
-
-	if read_err != nil {
-		return nil, read_err
-	}
-
-	return body, nil
-}
-
-func (c *Client) Login(login, password string) (Token, error) {
 	var token Token
 
 	data := url.Values{}
-	data.Add("login", login)
+	data.Set("login", login)
-	data.Add("password", password)
+	data.Set("password", password)
 
-	body, req_err := c.MakePostRequest(point_api_url + "login", data)
+	body, req_err := c.client.MakePostRequest(c.apiUrl+"login", data, nil)
 
 	if req_err != nil {
 		return token, req_err
@@ -128,7 +53,7 @@ func (c *Client) Login(login, password string) (Token, error) {
 	return token, nil
 }
 
-func (c *Client) GetRecentAllPostsPage() (Page, error) {
+func (c *PointClient) GetRecentAllPostsPage() (Page, error) {
 	var page Page
 
 	if len(c.token.AuthToken) == 0 {
@@ -136,7 +61,11 @@ func (c *Client) GetRecentAllPostsPage() (Page, error) {
 		return page, errors.New("Login first")
 	}
 
-	body, req_err := c.MakeGetRequest(point_api_url + "all", nil)
+	headers := map[string]string{
+		"Authorization": c.token.AuthToken,
+	}
+
+	body, req_err := c.client.MakeGetRequest(c.apiUrl+"all", nil, &headers)
 
 	if req_err != nil {
 		return page, errors.New("HTTP request error")
@@ -158,7 +87,7 @@ func (c *Client) GetRecentAllPostsPage() (Page, error) {
 	return page, nil
 }
 
-func (c *Client) GetNextAllPostsPage(page Page) (Page, error) {
+func (c *PointClient) GetNextAllPostsPage(page Page) (Page, error) {
 	var nextPage Page
 
 	if false == page.HasNext || len(page.Posts) == 0 {
@@ -171,9 +100,13 @@ func (c *Client) GetNextAllPostsPage(page Page) (Page, error) {
 	}
 
 	data := url.Values{}
-	data.Add("before", strconv.Itoa(page.Posts[len(page.Posts)-1].Uid))
+	data.Set("before", strconv.Itoa(page.Posts[len(page.Posts)-1].Uid))
 
-	body, req_err := c.MakeGetRequest(point_api_url + "all", &data)
+	headers := map[string]string{
+		"Authorization": c.token.AuthToken,
+	}
+
+	body, req_err := c.client.MakeGetRequest(c.apiUrl+"all", &data, &headers)
 
 	if req_err != nil {
 		return nextPage, errors.New("HTTP request error")
@@ -193,4 +126,4 @@ func (c *Client) GetNextAllPostsPage(page Page) (Page, error) {
 	}
 
 	return nextPage, nil
-}
+}

point/page.go

@@ -1,7 +1,7 @@
 package point
 
 type Page struct {
-	HasNext bool         `json:"has_next"`
+	HasNext bool       `json:"has_next"`
 	Posts   []MetaPost `json:"posts"`
-	Error   string       `json:"error"`
+	Error   string     `json:"error"`
 }

point_post_crawler.go

@@ -3,17 +3,12 @@ package main
 import (
 	"flag"
 	"fmt"
-	//"gopkg.in/yaml.v2"
-	//"net/http"
 	"bitbucket.org/skobkin/point-tools-crawler/point"
-	"time"
+	"bitbucket.org/skobkin/point-tools-crawler/point_tools"
 	"log"
-	"encoding/json"
+	"time"
-	"net/url"
 )
 
-
-
 func main() {
 	var login, password string
 
@@ -26,9 +21,10 @@ func main() {
 		return
 	}
 
-	client := point.Client{}
+	pointClient := point.NewClient("https://point.im/api/")
+	pointToolsClient := point_tools.NewClient("http://point-tools.local:8000/api/crawler/")
 
-	_, login_err := client.Login(login, password)
+	_, login_err := pointClient.Login(login, password)
 
 	if login_err != nil {
 		fmt.Printf("Login error %s", login_err)
@@ -37,7 +33,7 @@ func main() {
 		fmt.Printf("Successfully authenticated!\n")
 	}
 
-	page, req_err := client.GetRecentAllPostsPage()
+	page, req_err := pointClient.GetRecentAllPostsPage()
 
 	if req_err != nil {
 		log.Fatal(req_err)
@@ -45,72 +41,36 @@ func main() {
 	}
 
 	fmt.Printf("1 page requested\n")
-	printPageJSON(page)
-	sendPage(page)
 
-	pageNumber := 1;
+	if len(page.Posts) == 0 {
+		fmt.Println("Last uid", string(page.Posts[len(page.Posts)-1].Uid))
+	}
+
+	pointToolsClient.SendPage(page)
+
+	//var str string
+
+	pageNumber := 1
 
 	for page.HasNext {
 		pageNumber++
 
-		page, req_err = client.GetNextAllPostsPage(page)
+		page, req_err = pointClient.GetNextAllPostsPage(page)
 
 		if req_err != nil {
 			log.Fatal(req_err)
 			return
 		}
 
-		printPageJSON(page)
-		sendPage(page)
 
-		fmt.Printf("%d page requested\n", pageNumber)
+
+		fmt.Printf("%d page requested, last uid: %d", pageNumber)
 		fmt.Printf("%d posts\n", len(page.Posts))
 
+		pointToolsClient.SendPage(page)
+
 		time.Sleep(time.Second)
 	}
 }
 
-func getPageJSON(page point.Page) (string, error) {
-	b, err := json.Marshal(page)
 
-	if err != nil {
-		fmt.Println("error:", err)
-		return "", err
-	}
-
-	return string(b[:]), nil
-}
-
-func printPageJSON(page point.Page) {
-	json, err := getPageJSON(page)
-
-	if err != nil {
-		fmt.Println("JSON print error")
-	}
-
-	fmt.Println(json)
-}
-
-func sendPage(page point.Page) error {
-	json, err := getPageJSON(page)
-
-	if err != nil {
-		return err
-	}
-
-	client := point.Client{}
-
-	data := url.Values{}
-	data.Set("token", "test_token")
-	data.Add("json", json)
-
-	body, req_err := client.MakePostRequest("http://point-tools.local:8000/api/crawler/all/page", data)
-
-	if req_err != nil {
-		return req_err
-	}
-
-	fmt.Println("Response:", string(body[:]))
-
-	return nil
-}

point_tools/api_error.go

@@ -0,0 +1,6 @@
+package point_tools
+
+type ApiError struct {
+	Code    int    `json:"code"`
+	Message string `json:"message"`
+}

point_tools/client.go

@@ -0,0 +1,63 @@
+package point_tools
+
+import (
+	"bitbucket.org/skobkin/point-tools-crawler/simple_http"
+	"encoding/json"
+	"log"
+	"net/http"
+	"net/url"
+	"bitbucket.org/skobkin/point-tools-crawler/point"
+)
+
+type PointToolsClient struct {
+	client simple_http.Client
+	apiUrl string
+}
+
+func NewClient(apiUrl string) PointToolsClient {
+	return PointToolsClient{
+		simple_http.Client{http.Client{}},
+		apiUrl,
+	}
+}
+
+func GetPageJSON(page point.Page) (string, error) {
+	b, err := json.Marshal(page)
+
+	if err != nil {
+		log.Println("point.Page serialize error:", err)
+		return "", err
+	}
+
+	return string(b[:]), nil
+}
+
+func (c *PointToolsClient) SendPage(page point.Page) (ReceiveAllPageResponse, error) {
+	var response ReceiveAllPageResponse
+
+	jsonStr, err := GetPageJSON(page)
+
+	if err != nil {
+		return response, err
+	}
+
+	data := url.Values{}
+	data.Set("token", "test_token")
+	data.Add("json", jsonStr)
+
+	body, req_err := c.client.MakePostRequest(c.apiUrl+"all/page", data, nil)
+
+	if req_err != nil {
+		return response, req_err
+	}
+
+	json_err := json.Unmarshal(body, &response)
+
+	if json_err != nil {
+		return response, json_err
+	}
+
+	log.Println("Response:", string(body[:]))
+
+	return response, nil
+}

point_tools/receive_all_page_response.go

@@ -0,0 +1,11 @@
+package point_tools
+
+type ReceiveAllPageResponse struct {
+	Status string             `json:"status"`
+	Data   ReceiveAllPageData `json:"data"`
+	Error  ApiError           `json:"error"`
+}
+
+type ReceiveAllPageData struct {
+	Continue bool `json:"continue"`
+}

simple_http/client.go

@@ -0,0 +1,97 @@
+package simple_http
+
+import (
+	"io/ioutil"
+	"log"
+	"net/http"
+	"net/url"
+	"strings"
+)
+
+type Client struct {
+	HttpClient http.Client
+}
+
+func (c *Client) getResponseBody(resp *http.Response) ([]byte, error) {
+	body, read_err := ioutil.ReadAll(resp.Body)
+
+	if read_err != nil {
+		log.Fatal(read_err)
+		return nil, read_err
+	}
+
+	return body, nil
+}
+
+func (c *Client) MakeGetRequest(url string, params *url.Values, headers *map[string]string) ([]byte, error) {
+	var body []byte
+
+	if params != nil {
+		url += "?" + params.Encode()
+	}
+
+	req, req_err := http.NewRequest("GET", url, nil)
+
+	if req_err != nil {
+		log.Fatal(req_err)
+		return body, req_err
+	}
+
+	// Headers
+	if headers != nil {
+		for key, value := range *headers {
+			req.Header.Set(key, value)
+		}
+	}
+
+	resp, resp_err := c.HttpClient.Do(req)
+
+	if resp_err != nil {
+		log.Fatal(resp_err)
+		return nil, resp_err
+	}
+
+	defer resp.Body.Close()
+
+	body, read_err := c.getResponseBody(resp)
+
+	if read_err != nil {
+		return nil, read_err
+	}
+
+	return body, nil
+}
+
+func (c *Client) MakePostRequest(url string, params url.Values, headers *map[string]string) ([]byte, error) {
+	req, req_err := http.NewRequest("POST", url, strings.NewReader(params.Encode()))
+
+	if req_err != nil {
+		log.Fatal(req_err)
+		return nil, req_err
+	}
+
+	// Adding necessary for POST request header
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+
+	// Headers
+	if headers != nil {
+		for key, value := range *headers {
+			req.Header.Set(key, value)
+		}
+	}
+
+	resp, resp_err := c.HttpClient.Do(req)
+
+	if resp_err != nil {
+		log.Fatal(resp_err)
+		return nil, resp_err
+	}
+
+	body, read_err := c.getResponseBody(resp)
+
+	if read_err != nil {
+		return nil, read_err
+	}
+
+	return body, nil
+}