diff --git a/point/client.go b/point/client.go index a17a736..1898c51 100644 --- a/point/client.go +++ b/point/client.go @@ -1,112 +1,37 @@ package point import ( + "bitbucket.org/skobkin/point-tools-crawler/simple_http" + "encoding/json" + "errors" "log" "net/http" "net/url" - "io/ioutil" - "encoding/json" - "errors" - "strings" "strconv" ) -const point_api_url string = "https://point.im/api/" - -type Client struct { - token Token +type PointClient struct { + client simple_http.Client + apiUrl string + token Token } -func (c *Client) getResponseBody(resp *http.Response) ([]byte, error) { - body, read_err := ioutil.ReadAll(resp.Body) - - if read_err != nil { - log.Fatal(read_err) - return nil, read_err +func NewClient(apiUrl string) PointClient { + return PointClient{ + simple_http.Client{http.Client{}}, + apiUrl, + Token{}, } - - return body, nil } -func (c *Client) MakeGetRequest(url string, params *url.Values) ([]byte, error) { - var body []byte - - if params != nil { - url += "?" + params.Encode() - } - - req, req_err := http.NewRequest("GET", url, nil) - - if req_err != nil { - log.Fatal(req_err) - return body, req_err - } - - if len(c.token.AuthToken) > 0 { - req.Header.Set("Authorization", c.token.AuthToken) - } - - hc := &http.Client{} - - resp, resp_err := hc.Do(req) - - if resp_err != nil { - log.Fatal(resp_err) - return nil, resp_err - } - - defer resp.Body.Close() - - body, read_err := c.getResponseBody(resp) - - if read_err != nil { - return nil, read_err - } - - return body, nil -} - -func (c *Client) MakePostRequest(url string, params url.Values) ([]byte, error) { - req, req_err := http.NewRequest("POST", url, strings.NewReader(params.Encode())) - - if req_err != nil { - log.Fatal(req_err) - return nil, req_err - } - - hc := &http.Client{} - - if len(c.token.AuthToken) > 0 { - req.Header.Set("Authorization", c.token.AuthToken) - } - - // Adding necessary for POST request header - req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - - resp, resp_err := hc.Do(req) - - if resp_err != nil { - log.Fatal(resp_err) - return nil, resp_err - } - - body, read_err := c.getResponseBody(resp) - - if read_err != nil { - return nil, read_err - } - - return body, nil -} - -func (c *Client) Login(login, password string) (Token, error) { +func (c *PointClient) Login(login, password string) (Token, error) { var token Token data := url.Values{} - data.Add("login", login) - data.Add("password", password) + data.Set("login", login) + data.Set("password", password) - body, req_err := c.MakePostRequest(point_api_url + "login", data) + body, req_err := c.client.MakePostRequest(c.apiUrl+"login", data, nil) if req_err != nil { return token, req_err @@ -128,7 +53,7 @@ func (c *Client) Login(login, password string) (Token, error) { return token, nil } -func (c *Client) GetRecentAllPostsPage() (Page, error) { +func (c *PointClient) GetRecentAllPostsPage() (Page, error) { var page Page if len(c.token.AuthToken) == 0 { @@ -136,7 +61,11 @@ func (c *Client) GetRecentAllPostsPage() (Page, error) { return page, errors.New("Login first") } - body, req_err := c.MakeGetRequest(point_api_url + "all", nil) + headers := map[string]string{ + "Authorization": c.token.AuthToken, + } + + body, req_err := c.client.MakeGetRequest(c.apiUrl+"all", nil, &headers) if req_err != nil { return page, errors.New("HTTP request error") @@ -158,7 +87,7 @@ func (c *Client) GetRecentAllPostsPage() (Page, error) { return page, nil } -func (c *Client) GetNextAllPostsPage(page Page) (Page, error) { +func (c *PointClient) GetNextAllPostsPage(page Page) (Page, error) { var nextPage Page if false == page.HasNext || len(page.Posts) == 0 { @@ -171,9 +100,13 @@ func (c *Client) GetNextAllPostsPage(page Page) (Page, error) { } data := url.Values{} - data.Add("before", strconv.Itoa(page.Posts[len(page.Posts)-1].Uid)) + data.Set("before", strconv.Itoa(page.Posts[len(page.Posts)-1].Uid)) - body, req_err := c.MakeGetRequest(point_api_url + "all", &data) + headers := map[string]string{ + "Authorization": c.token.AuthToken, + } + + body, req_err := c.client.MakeGetRequest(c.apiUrl+"all", &data, &headers) if req_err != nil { return nextPage, errors.New("HTTP request error") @@ -193,4 +126,4 @@ func (c *Client) GetNextAllPostsPage(page Page) (Page, error) { } return nextPage, nil -} \ No newline at end of file +} diff --git a/point/page.go b/point/page.go index 8d5b0ee..3e304b4 100644 --- a/point/page.go +++ b/point/page.go @@ -1,7 +1,7 @@ package point type Page struct { - HasNext bool `json:"has_next"` + HasNext bool `json:"has_next"` Posts []MetaPost `json:"posts"` - Error string `json:"error"` + Error string `json:"error"` } diff --git a/point_post_crawler.go b/point_post_crawler.go index b9dca0a..153de3a 100644 --- a/point_post_crawler.go +++ b/point_post_crawler.go @@ -3,17 +3,12 @@ package main import ( "flag" "fmt" - //"gopkg.in/yaml.v2" - //"net/http" "bitbucket.org/skobkin/point-tools-crawler/point" - "time" + "bitbucket.org/skobkin/point-tools-crawler/point_tools" "log" - "encoding/json" - "net/url" + "time" ) - - func main() { var login, password string @@ -26,9 +21,10 @@ func main() { return } - client := point.Client{} + pointClient := point.NewClient("https://point.im/api/") + pointToolsClient := point_tools.NewClient("http://point-tools.local:8000/api/crawler/") - _, login_err := client.Login(login, password) + _, login_err := pointClient.Login(login, password) if login_err != nil { fmt.Printf("Login error %s", login_err) @@ -37,7 +33,7 @@ func main() { fmt.Printf("Successfully authenticated!\n") } - page, req_err := client.GetRecentAllPostsPage() + page, req_err := pointClient.GetRecentAllPostsPage() if req_err != nil { log.Fatal(req_err) @@ -45,72 +41,36 @@ func main() { } fmt.Printf("1 page requested\n") - printPageJSON(page) - sendPage(page) - pageNumber := 1; + if len(page.Posts) == 0 { + fmt.Println("Last uid", string(page.Posts[len(page.Posts)-1].Uid)) + } + + pointToolsClient.SendPage(page) + + //var str string + + pageNumber := 1 for page.HasNext { pageNumber++ - page, req_err = client.GetNextAllPostsPage(page) + page, req_err = pointClient.GetNextAllPostsPage(page) if req_err != nil { log.Fatal(req_err) return } - printPageJSON(page) - sendPage(page) - fmt.Printf("%d page requested\n", pageNumber) + + fmt.Printf("%d page requested, last uid: %d", pageNumber) fmt.Printf("%d posts\n", len(page.Posts)) + pointToolsClient.SendPage(page) + time.Sleep(time.Second) } } -func getPageJSON(page point.Page) (string, error) { - b, err := json.Marshal(page) - if err != nil { - fmt.Println("error:", err) - return "", err - } - - return string(b[:]), nil -} - -func printPageJSON(page point.Page) { - json, err := getPageJSON(page) - - if err != nil { - fmt.Println("JSON print error") - } - - fmt.Println(json) -} - -func sendPage(page point.Page) error { - json, err := getPageJSON(page) - - if err != nil { - return err - } - - client := point.Client{} - - data := url.Values{} - data.Set("token", "test_token") - data.Add("json", json) - - body, req_err := client.MakePostRequest("http://point-tools.local:8000/api/crawler/all/page", data) - - if req_err != nil { - return req_err - } - - fmt.Println("Response:", string(body[:])) - - return nil -} \ No newline at end of file diff --git a/point_tools/api_error.go b/point_tools/api_error.go new file mode 100644 index 0000000..04bef96 --- /dev/null +++ b/point_tools/api_error.go @@ -0,0 +1,6 @@ +package point_tools + +type ApiError struct { + Code int `json:"code"` + Message string `json:"message"` +} diff --git a/point_tools/client.go b/point_tools/client.go new file mode 100644 index 0000000..9af246d --- /dev/null +++ b/point_tools/client.go @@ -0,0 +1,63 @@ +package point_tools + +import ( + "bitbucket.org/skobkin/point-tools-crawler/simple_http" + "encoding/json" + "log" + "net/http" + "net/url" + "bitbucket.org/skobkin/point-tools-crawler/point" +) + +type PointToolsClient struct { + client simple_http.Client + apiUrl string +} + +func NewClient(apiUrl string) PointToolsClient { + return PointToolsClient{ + simple_http.Client{http.Client{}}, + apiUrl, + } +} + +func GetPageJSON(page point.Page) (string, error) { + b, err := json.Marshal(page) + + if err != nil { + log.Println("point.Page serialize error:", err) + return "", err + } + + return string(b[:]), nil +} + +func (c *PointToolsClient) SendPage(page point.Page) (ReceiveAllPageResponse, error) { + var response ReceiveAllPageResponse + + jsonStr, err := GetPageJSON(page) + + if err != nil { + return response, err + } + + data := url.Values{} + data.Set("token", "test_token") + data.Add("json", jsonStr) + + body, req_err := c.client.MakePostRequest(c.apiUrl+"all/page", data, nil) + + if req_err != nil { + return response, req_err + } + + json_err := json.Unmarshal(body, &response) + + if json_err != nil { + return response, json_err + } + + log.Println("Response:", string(body[:])) + + return response, nil +} \ No newline at end of file diff --git a/point_tools/receive_all_page_response.go b/point_tools/receive_all_page_response.go new file mode 100644 index 0000000..b8c6ee0 --- /dev/null +++ b/point_tools/receive_all_page_response.go @@ -0,0 +1,11 @@ +package point_tools + +type ReceiveAllPageResponse struct { + Status string `json:"status"` + Data ReceiveAllPageData `json:"data"` + Error ApiError `json:"error"` +} + +type ReceiveAllPageData struct { + Continue bool `json:"continue"` +} diff --git a/simple_http/client.go b/simple_http/client.go new file mode 100644 index 0000000..f93a876 --- /dev/null +++ b/simple_http/client.go @@ -0,0 +1,97 @@ +package simple_http + +import ( + "io/ioutil" + "log" + "net/http" + "net/url" + "strings" +) + +type Client struct { + HttpClient http.Client +} + +func (c *Client) getResponseBody(resp *http.Response) ([]byte, error) { + body, read_err := ioutil.ReadAll(resp.Body) + + if read_err != nil { + log.Fatal(read_err) + return nil, read_err + } + + return body, nil +} + +func (c *Client) MakeGetRequest(url string, params *url.Values, headers *map[string]string) ([]byte, error) { + var body []byte + + if params != nil { + url += "?" + params.Encode() + } + + req, req_err := http.NewRequest("GET", url, nil) + + if req_err != nil { + log.Fatal(req_err) + return body, req_err + } + + // Headers + if headers != nil { + for key, value := range *headers { + req.Header.Set(key, value) + } + } + + resp, resp_err := c.HttpClient.Do(req) + + if resp_err != nil { + log.Fatal(resp_err) + return nil, resp_err + } + + defer resp.Body.Close() + + body, read_err := c.getResponseBody(resp) + + if read_err != nil { + return nil, read_err + } + + return body, nil +} + +func (c *Client) MakePostRequest(url string, params url.Values, headers *map[string]string) ([]byte, error) { + req, req_err := http.NewRequest("POST", url, strings.NewReader(params.Encode())) + + if req_err != nil { + log.Fatal(req_err) + return nil, req_err + } + + // Adding necessary for POST request header + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + // Headers + if headers != nil { + for key, value := range *headers { + req.Header.Set(key, value) + } + } + + resp, resp_err := c.HttpClient.Do(req) + + if resp_err != nil { + log.Fatal(resp_err) + return nil, resp_err + } + + body, read_err := c.getResponseBody(resp) + + if read_err != nil { + return nil, read_err + } + + return body, nil +}