skobkin
/
point-tools-crawler
mirror of https://bitbucket.org/skobkin/point-tools-crawler.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

HTTP client functionality refactored. PointToolsClient added.

This commit is contained in:
Alexey Skobkin 2016-03-21 01:06:34 +03:00
parent 9e1adb2867
commit 24f532d6f1
7 changed files with 229 additions and 159 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

127
point/client.go
View File

@ -1,112 +1,37 @@
package point
import (
"bitbucket.org/skobkin/point-tools-crawler/simple_http"
"encoding/json"
"errors"
"log"
"net/http"
"net/url"
"io/ioutil"
"encoding/json"
"errors"
"strings"
"strconv"
)
const point_api_url string = "https://point.im/api/"
type Client struct {
token Token
type PointClient struct {
client simple_http.Client
apiUrl string
token Token
}
func (c *Client) getResponseBody(resp *http.Response) ([]byte, error) {
body, read_err := ioutil.ReadAll(resp.Body)
if read_err != nil {
log.Fatal(read_err)
return nil, read_err
func NewClient(apiUrl string) PointClient {
return PointClient{
simple_http.Client{http.Client{}},
apiUrl,
Token{},
}
return body, nil
}
func (c *Client) MakeGetRequest(url string, params *url.Values) ([]byte, error) {
var body []byte
if params != nil {
url += "?" + params.Encode()
}
req, req_err := http.NewRequest("GET", url, nil)
if req_err != nil {
log.Fatal(req_err)
return body, req_err
}
if len(c.token.AuthToken) > 0 {
req.Header.Set("Authorization", c.token.AuthToken)
}
hc := &http.Client{}
resp, resp_err := hc.Do(req)
if resp_err != nil {
log.Fatal(resp_err)
return nil, resp_err
}
defer resp.Body.Close()
body, read_err := c.getResponseBody(resp)
if read_err != nil {
return nil, read_err
}
return body, nil
}
func (c *Client) MakePostRequest(url string, params url.Values) ([]byte, error) {
req, req_err := http.NewRequest("POST", url, strings.NewReader(params.Encode()))
if req_err != nil {
log.Fatal(req_err)
return nil, req_err
}
hc := &http.Client{}
if len(c.token.AuthToken) > 0 {
req.Header.Set("Authorization", c.token.AuthToken)
}
// Adding necessary for POST request header
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
resp, resp_err := hc.Do(req)
if resp_err != nil {
log.Fatal(resp_err)
return nil, resp_err
}
body, read_err := c.getResponseBody(resp)
if read_err != nil {
return nil, read_err
}
return body, nil
}
func (c *Client) Login(login, password string) (Token, error) {
func (c *PointClient) Login(login, password string) (Token, error) {
var token Token
data := url.Values{}
data.Add("login", login)
data.Add("password", password)
data.Set("login", login)
data.Set("password", password)
body, req_err := c.MakePostRequest(point_api_url + "login", data)
body, req_err := c.client.MakePostRequest(c.apiUrl+"login", data, nil)
if req_err != nil {
return token, req_err
@ -128,7 +53,7 @@ func (c *Client) Login(login, password string) (Token, error) {
return token, nil
}
func (c *Client) GetRecentAllPostsPage() (Page, error) {
func (c *PointClient) GetRecentAllPostsPage() (Page, error) {
var page Page
if len(c.token.AuthToken) == 0 {
@ -136,7 +61,11 @@ func (c *Client) GetRecentAllPostsPage() (Page, error) {
return page, errors.New("Login first")
}
body, req_err := c.MakeGetRequest(point_api_url + "all", nil)
headers := map[string]string{
"Authorization": c.token.AuthToken,
}
body, req_err := c.client.MakeGetRequest(c.apiUrl+"all", nil, &headers)
if req_err != nil {
return page, errors.New("HTTP request error")
@ -158,7 +87,7 @@ func (c *Client) GetRecentAllPostsPage() (Page, error) {
return page, nil
}
func (c *Client) GetNextAllPostsPage(page Page) (Page, error) {
func (c *PointClient) GetNextAllPostsPage(page Page) (Page, error) {
var nextPage Page
if false == page.HasNext || len(page.Posts) == 0 {
@ -171,9 +100,13 @@ func (c *Client) GetNextAllPostsPage(page Page) (Page, error) {
}
data := url.Values{}
data.Add("before", strconv.Itoa(page.Posts[len(page.Posts)-1].Uid))
data.Set("before", strconv.Itoa(page.Posts[len(page.Posts)-1].Uid))
body, req_err := c.MakeGetRequest(point_api_url + "all", &data)
headers := map[string]string{
"Authorization": c.token.AuthToken,
}
body, req_err := c.client.MakeGetRequest(c.apiUrl+"all", &data, &headers)
if req_err != nil {
return nextPage, errors.New("HTTP request error")
@ -193,4 +126,4 @@ func (c *Client) GetNextAllPostsPage(page Page) (Page, error) {
}
return nextPage, nil
}
}

4
point/page.go
View File

@ -1,7 +1,7 @@
package point
type Page struct {
HasNext bool `json:"has_next"`
HasNext bool `json:"has_next"`
Posts []MetaPost `json:"posts"`
Error string `json:"error"`
Error string `json:"error"`
}

80
point_post_crawler.go
View File

@ -3,17 +3,12 @@ package main
import (
"flag"
"fmt"
//"gopkg.in/yaml.v2"
//"net/http"
"bitbucket.org/skobkin/point-tools-crawler/point"
"time"
"bitbucket.org/skobkin/point-tools-crawler/point_tools"
"log"
"encoding/json"
"net/url"
"time"
)
func main() {
var login, password string
@ -26,9 +21,10 @@ func main() {
return
}
client := point.Client{}
pointClient := point.NewClient("https://point.im/api/")
pointToolsClient := point_tools.NewClient("http://point-tools.local:8000/api/crawler/")
_, login_err := client.Login(login, password)
_, login_err := pointClient.Login(login, password)
if login_err != nil {
fmt.Printf("Login error %s", login_err)
@ -37,7 +33,7 @@ func main() {
fmt.Printf("Successfully authenticated!\n")
}
page, req_err := client.GetRecentAllPostsPage()
page, req_err := pointClient.GetRecentAllPostsPage()
if req_err != nil {
log.Fatal(req_err)
@ -45,72 +41,36 @@ func main() {
}
fmt.Printf("1 page requested\n")
printPageJSON(page)
sendPage(page)
pageNumber := 1;
if len(page.Posts) == 0 {
fmt.Println("Last uid", string(page.Posts[len(page.Posts)-1].Uid))
}
pointToolsClient.SendPage(page)
//var str string
pageNumber := 1
for page.HasNext {
pageNumber++
page, req_err = client.GetNextAllPostsPage(page)
page, req_err = pointClient.GetNextAllPostsPage(page)
if req_err != nil {
log.Fatal(req_err)
return
}
printPageJSON(page)
sendPage(page)
fmt.Printf("%d page requested\n", pageNumber)
fmt.Printf("%d page requested, last uid: %d", pageNumber)
fmt.Printf("%d posts\n", len(page.Posts))
pointToolsClient.SendPage(page)
time.Sleep(time.Second)
}
}
func getPageJSON(page point.Page) (string, error) {
b, err := json.Marshal(page)
if err != nil {
fmt.Println("error:", err)
return "", err
}
return string(b[:]), nil
}
func printPageJSON(page point.Page) {
json, err := getPageJSON(page)
if err != nil {
fmt.Println("JSON print error")
}
fmt.Println(json)
}
func sendPage(page point.Page) error {
json, err := getPageJSON(page)
if err != nil {
return err
}
client := point.Client{}
data := url.Values{}
data.Set("token", "test_token")
data.Add("json", json)
body, req_err := client.MakePostRequest("http://point-tools.local:8000/api/crawler/all/page", data)
if req_err != nil {
return req_err
}
fmt.Println("Response:", string(body[:]))
return nil
}

6
point_tools/api_error.go Normal file
View File

@ -0,0 +1,6 @@
package point_tools
type ApiError struct {
Code int `json:"code"`
Message string `json:"message"`
}

63
point_tools/client.go Normal file
View File

@ -0,0 +1,63 @@
package point_tools
import (
"bitbucket.org/skobkin/point-tools-crawler/simple_http"
"encoding/json"
"log"
"net/http"
"net/url"
"bitbucket.org/skobkin/point-tools-crawler/point"
)
type PointToolsClient struct {
client simple_http.Client
apiUrl string
}
func NewClient(apiUrl string) PointToolsClient {
return PointToolsClient{
simple_http.Client{http.Client{}},
apiUrl,
}
}
func GetPageJSON(page point.Page) (string, error) {
b, err := json.Marshal(page)
if err != nil {
log.Println("point.Page serialize error:", err)
return "", err
}
return string(b[:]), nil
}
func (c *PointToolsClient) SendPage(page point.Page) (ReceiveAllPageResponse, error) {
var response ReceiveAllPageResponse
jsonStr, err := GetPageJSON(page)
if err != nil {
return response, err
}
data := url.Values{}
data.Set("token", "test_token")
data.Add("json", jsonStr)
body, req_err := c.client.MakePostRequest(c.apiUrl+"all/page", data, nil)
if req_err != nil {
return response, req_err
}
json_err := json.Unmarshal(body, &response)
if json_err != nil {
return response, json_err
}
log.Println("Response:", string(body[:]))
return response, nil
}

11
point_tools/receive_all_page_response.go Normal file
View File

@ -0,0 +1,11 @@
package point_tools
type ReceiveAllPageResponse struct {
Status string `json:"status"`
Data ReceiveAllPageData `json:"data"`
Error ApiError `json:"error"`
}
type ReceiveAllPageData struct {
Continue bool `json:"continue"`
}

97
simple_http/client.go Normal file
View File

@ -0,0 +1,97 @@
package simple_http
import (
"io/ioutil"
"log"
"net/http"
"net/url"
"strings"
)
type Client struct {
HttpClient http.Client
}
func (c *Client) getResponseBody(resp *http.Response) ([]byte, error) {
body, read_err := ioutil.ReadAll(resp.Body)
if read_err != nil {
log.Fatal(read_err)
return nil, read_err
}
return body, nil
}
func (c *Client) MakeGetRequest(url string, params *url.Values, headers *map[string]string) ([]byte, error) {
var body []byte
if params != nil {
url += "?" + params.Encode()
}
req, req_err := http.NewRequest("GET", url, nil)
if req_err != nil {
log.Fatal(req_err)
return body, req_err
}
// Headers
if headers != nil {
for key, value := range *headers {
req.Header.Set(key, value)
}
}
resp, resp_err := c.HttpClient.Do(req)
if resp_err != nil {
log.Fatal(resp_err)
return nil, resp_err
}
defer resp.Body.Close()
body, read_err := c.getResponseBody(resp)
if read_err != nil {
return nil, read_err
}
return body, nil
}
func (c *Client) MakePostRequest(url string, params url.Values, headers *map[string]string) ([]byte, error) {
req, req_err := http.NewRequest("POST", url, strings.NewReader(params.Encode()))
if req_err != nil {
log.Fatal(req_err)
return nil, req_err
}
// Adding necessary for POST request header
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
// Headers
if headers != nil {
for key, value := range *headers {
req.Header.Set(key, value)
}
}
resp, resp_err := c.HttpClient.Do(req)
if resp_err != nil {
log.Fatal(resp_err)
return nil, resp_err
}
body, read_err := c.getResponseBody(resp)
if read_err != nil {
return nil, read_err
}
return body, nil
}