mirror of
https://bitbucket.org/skobkin/point-tools-crawler.git
synced 2024-12-05 02:25:53 +00:00
Code separated. Point Tools and Point.im API wrappers moved to separate repositories.
This commit is contained in:
Alexey Skobkin
parent
9efce351d8
commit
8c5c4cf430
142
point/client.go
142
point/client.go
|
|
@ -1,142 +0,0 @@
|
|||
package point
|
||||
|
||||
import (
|
||||
"bitbucket.org/skobkin/point-tools-crawler/simple_http"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
type PointClient struct {
|
||||
client simple_http.Client
|
||||
apiUrl string
|
||||
token Token
|
||||
}
|
||||
|
||||
func NewClient(apiUrl string) PointClient {
|
||||
return PointClient{
|
||||
simple_http.Client{http.Client{}},
|
||||
apiUrl,
|
||||
Token{},
|
||||
}
|
||||
}
|
||||
|
||||
func (c *PointClient) Login(login, password string) (Token, error) {
|
||||
var token Token
|
||||
|
||||
data := url.Values{}
|
||||
data.Set("login", login)
|
||||
data.Set("password", password)
|
||||
|
||||
body, reqErr := c.client.MakePostRequest(c.apiUrl+"login", data, nil)
|
||||
|
||||
if reqErr != nil {
|
||||
return token, reqErr
|
||||
}
|
||||
|
||||
jsonErr := json.Unmarshal(body, &token)
|
||||
|
||||
if jsonErr != nil {
|
||||
log.Fatal(jsonErr)
|
||||
return token, errors.New("JSON deserialization error")
|
||||
}
|
||||
|
||||
if token.Error != "" {
|
||||
return token, errors.New(token.Error)
|
||||
}
|
||||
|
||||
c.token = token
|
||||
|
||||
return token, nil
|
||||
}
|
||||
|
||||
func (c *PointClient) GetRecentAllPostsPage() (Page, error) {
|
||||
var page Page
|
||||
|
||||
if len(c.token.AuthToken) == 0 {
|
||||
log.Fatal("Can not get recent posts. Login first.")
|
||||
return page, errors.New("Login first")
|
||||
}
|
||||
|
||||
headers := map[string]string{
|
||||
"Authorization": c.token.AuthToken,
|
||||
}
|
||||
|
||||
body, reqErr := c.client.MakeGetRequest(c.apiUrl+"all", nil, &headers)
|
||||
|
||||
if reqErr != nil {
|
||||
return page, errors.New("HTTP request error")
|
||||
}
|
||||
|
||||
jsonErr := json.Unmarshal(body, &page)
|
||||
|
||||
if jsonErr != nil {
|
||||
log.Fatal(jsonErr)
|
||||
return page, errors.New("JSON deserialization error")
|
||||
}
|
||||
|
||||
// Move page.Error to response object
|
||||
if page.Error == "NotAuthorized" {
|
||||
log.Fatal("Please use correct login and password to fetch recent posts")
|
||||
return page, errors.New("Not authorized")
|
||||
}
|
||||
|
||||
return page, nil
|
||||
}
|
||||
|
||||
func (c *PointClient) GetNextAllPostsPageBeforeUid(uid int) (Page, error) {
|
||||
var nextPage Page
|
||||
|
||||
if len(c.token.AuthToken) == 0 {
|
||||
log.Fatal("Can not get recent posts. Login first.")
|
||||
return nextPage, errors.New("Login first")
|
||||
}
|
||||
|
||||
data := url.Values{}
|
||||
data.Set("before", strconv.Itoa(uid))
|
||||
|
||||
headers := map[string]string{
|
||||
"Authorization": c.token.AuthToken,
|
||||
}
|
||||
|
||||
body, req_err := c.client.MakeGetRequest(c.apiUrl+"all", &data, &headers)
|
||||
|
||||
if req_err != nil {
|
||||
return nextPage, errors.New("HTTP request error")
|
||||
}
|
||||
|
||||
json_err := json.Unmarshal(body, &nextPage)
|
||||
|
||||
if json_err != nil {
|
||||
log.Fatal(json_err)
|
||||
return nextPage, errors.New("JSON deserialization error")
|
||||
}
|
||||
|
||||
// Move page.Error to response object
|
||||
if nextPage.Error == "NotAuthorized" {
|
||||
log.Fatal("Please use correct login and password to fetch recent posts")
|
||||
return nextPage, errors.New("Not authorized")
|
||||
}
|
||||
|
||||
return nextPage, nil
|
||||
}
|
||||
|
||||
func (c *PointClient) GetNextAllPostsPage(page Page) (Page, error) {
|
||||
var nextPage Page
|
||||
|
||||
if len(page.Posts) == 0 {
|
||||
return nextPage, errors.New("Page must have has_next=true and some posts")
|
||||
}
|
||||
|
||||
if len(c.token.AuthToken) == 0 {
|
||||
log.Fatal("Can not get recent posts. Login first.")
|
||||
return nextPage, errors.New("Login first")
|
||||
}
|
||||
|
||||
nextPage, err := c.GetNextAllPostsPageBeforeUid(page.Posts[len(page.Posts)-1].Uid)
|
||||
|
||||
return nextPage, err
|
||||
}
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
package point
|
||||
|
||||
type MetaPost struct {
|
||||
Bookmarked bool `json:"bookmarked"`
|
||||
Uid int `json:"uid"`
|
||||
Subscribed bool `json:"subscribed"`
|
||||
Editable bool `json:"editable"`
|
||||
Recommended bool `json:"recommended"`
|
||||
Post Post `json:"post"`
|
||||
}
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
package point
|
||||
|
||||
type Page struct {
|
||||
HasNext bool `json:"has_next"`
|
||||
Posts []MetaPost `json:"posts"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
package point
|
||||
|
||||
type Post struct {
|
||||
Id string `json:"id"`
|
||||
Author User `json:"author"`
|
||||
Tags []string `json:"tags"`
|
||||
Text string `json:"text"`
|
||||
Files []string `json:"files"`
|
||||
CommentsCount int `json:"comments_count"`
|
||||
Created string `json:"created"`
|
||||
Pinned bool `json:"pinned"`
|
||||
Private bool `json:"private"`
|
||||
}
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
package point
|
||||
|
||||
type Token struct {
|
||||
AuthToken string `json:"token"`
|
||||
CsRfToken string `json:"csrf_token"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
package point
|
||||
|
||||
type User struct {
|
||||
Id int `json:"id"`
|
||||
Login string `json:"login"`
|
||||
}
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bitbucket.org/skobkin/point-tools-crawler/point"
|
||||
"bitbucket.org/skobkin/point-tools-crawler/point_tools"
|
||||
"bitbucket.org/skobkin/point-api-go"
|
||||
"bitbucket.org/skobkin/point-tools-go"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
|
|
@ -37,8 +37,8 @@ func main() {
|
|||
return
|
||||
}
|
||||
|
||||
pointClient := point.NewClient(pointApiUrl)
|
||||
pointToolsClient := point_tools.NewClient(pointToolsApiUrl, pointToolsToken)
|
||||
pointClient := point.New(pointApiUrl)
|
||||
pointToolsClient := point_tools.New(pointToolsApiUrl, pointToolsToken)
|
||||
|
||||
_, loginErr := pointClient.Login(pointLogin, pointPassword)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +0,0 @@
|
|||
package point_tools
|
||||
|
||||
type ApiError struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
|
@ -1,63 +0,0 @@
|
|||
package point_tools
|
||||
|
||||
import (
|
||||
"bitbucket.org/skobkin/point-tools-crawler/point"
|
||||
"bitbucket.org/skobkin/point-tools-crawler/simple_http"
|
||||
"encoding/json"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
)
|
||||
|
||||
type PointToolsClient struct {
|
||||
client simple_http.Client
|
||||
apiUrl string
|
||||
token string
|
||||
}
|
||||
|
||||
func NewClient(apiUrl, token string) PointToolsClient {
|
||||
return PointToolsClient{
|
||||
simple_http.Client{http.Client{}},
|
||||
apiUrl,
|
||||
token,
|
||||
}
|
||||
}
|
||||
|
||||
func GetPageJSON(page point.Page) (string, error) {
|
||||
b, err := json.Marshal(page)
|
||||
|
||||
if err != nil {
|
||||
log.Println("point.Page serialize error:", err)
|
||||
return "", err
|
||||
}
|
||||
|
||||
return string(b[:]), nil
|
||||
}
|
||||
|
||||
func (c *PointToolsClient) SendPage(page point.Page) (ReceiveAllPageResponse, error) {
|
||||
var response ReceiveAllPageResponse
|
||||
|
||||
jsonStr, err := GetPageJSON(page)
|
||||
|
||||
if err != nil {
|
||||
return response, err
|
||||
}
|
||||
|
||||
data := url.Values{}
|
||||
data.Set("token", c.token)
|
||||
data.Add("json", jsonStr)
|
||||
|
||||
body, reqErr := c.client.MakePostRequest(c.apiUrl+"all/page", data, nil)
|
||||
|
||||
if reqErr != nil {
|
||||
return response, reqErr
|
||||
}
|
||||
|
||||
jsonErr := json.Unmarshal(body, &response)
|
||||
|
||||
if jsonErr != nil {
|
||||
return response, jsonErr
|
||||
}
|
||||
|
||||
return response, nil
|
||||
}
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
package point_tools
|
||||
|
||||
const (
|
||||
STATUS_SUCCESS string = "success"
|
||||
STATUS_FAILURE string = "fail"
|
||||
)
|
||||
|
||||
type ReceiveAllPageResponse struct {
|
||||
Status string `json:"status"`
|
||||
Data ReceiveAllPageData `json:"data"`
|
||||
Error ApiError `json:"error"`
|
||||
}
|
||||
|
||||
type ReceiveAllPageData struct {
|
||||
Continue bool `json:"continue"`
|
||||
}
|
||||
|
|
@ -1,97 +0,0 @@
|
|||
package simple_http
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Client struct {
|
||||
HttpClient http.Client
|
||||
}
|
||||
|
||||
func (c *Client) getResponseBody(resp *http.Response) ([]byte, error) {
|
||||
body, readErr := ioutil.ReadAll(resp.Body)
|
||||
|
||||
if readErr != nil {
|
||||
log.Fatal(readErr)
|
||||
return nil, readErr
|
||||
}
|
||||
|
||||
return body, nil
|
||||
}
|
||||
|
||||
func (c *Client) MakeGetRequest(url string, params *url.Values, headers *map[string]string) ([]byte, error) {
|
||||
var body []byte
|
||||
|
||||
if params != nil {
|
||||
url += "?" + params.Encode()
|
||||
}
|
||||
|
||||
req, reqErr := http.NewRequest("GET", url, nil)
|
||||
|
||||
if reqErr != nil {
|
||||
log.Fatal(reqErr)
|
||||
return body, reqErr
|
||||
}
|
||||
|
||||
// Headers
|
||||
if headers != nil {
|
||||
for key, value := range *headers {
|
||||
req.Header.Set(key, value)
|
||||
}
|
||||
}
|
||||
|
||||
resp, respErr := c.HttpClient.Do(req)
|
||||
|
||||
if respErr != nil {
|
||||
log.Fatal(respErr)
|
||||
return nil, respErr
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, readErr := c.getResponseBody(resp)
|
||||
|
||||
if readErr != nil {
|
||||
return nil, readErr
|
||||
}
|
||||
|
||||
return body, nil
|
||||
}
|
||||
|
||||
func (c *Client) MakePostRequest(url string, params url.Values, headers *map[string]string) ([]byte, error) {
|
||||
req, reqErr := http.NewRequest("POST", url, strings.NewReader(params.Encode()))
|
||||
|
||||
if reqErr != nil {
|
||||
log.Fatal(reqErr)
|
||||
return nil, reqErr
|
||||
}
|
||||
|
||||
// Adding necessary for POST request header
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
|
||||
// Headers
|
||||
if headers != nil {
|
||||
for key, value := range *headers {
|
||||
req.Header.Set(key, value)
|
||||
}
|
||||
}
|
||||
|
||||
resp, respErr := c.HttpClient.Do(req)
|
||||
|
||||
if respErr != nil {
|
||||
log.Fatal(respErr)
|
||||
return nil, respErr
|
||||
}
|
||||
|
||||
body, readErr := c.getResponseBody(resp)
|
||||
|
||||
if readErr != nil {
|
||||
return nil, readErr
|
||||
}
|
||||
|
||||
return body, nil
|
||||
}
|
||||
Loading…
Reference in a new issue