skobkin/point-tools-crawler
1
0
Fork 0
mirror of https://bitbucket.org/skobkin/point-tools-crawler.git synced 2024-12-04 18:25:52 +00:00
Code Issues Projects Releases Wiki Activity

Code separated. Point Tools and Point.im API wrappers moved to separate repositories.

Browse source
This commit is contained in:
Alexey Skobkin 2016-03-29 10:32:20 +03:00
parent 9efce351d8
commit 8c5c4cf430
11 changed files with 4 additions and 371 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

142
point/client.go
View file

@ -1,142 +0,0 @@
package point
import (
"bitbucket.org/skobkin/point-tools-crawler/simple_http"
"encoding/json"
"errors"
"log"
"net/http"
"net/url"
"strconv"
)
type PointClient struct {
client simple_http.Client
apiUrl string
token Token
}
func NewClient(apiUrl string) PointClient {
return PointClient{
simple_http.Client{http.Client{}},
apiUrl,
Token{},
}
}
func (c *PointClient) Login(login, password string) (Token, error) {
var token Token
data := url.Values{}
data.Set("login", login)
data.Set("password", password)
body, reqErr := c.client.MakePostRequest(c.apiUrl+"login", data, nil)
if reqErr != nil {
return token, reqErr
}
jsonErr := json.Unmarshal(body, &token)
if jsonErr != nil {
log.Fatal(jsonErr)
return token, errors.New("JSON deserialization error")
}
if token.Error != "" {
return token, errors.New(token.Error)
}
c.token = token
return token, nil
}
func (c *PointClient) GetRecentAllPostsPage() (Page, error) {
var page Page
if len(c.token.AuthToken) == 0 {
log.Fatal("Can not get recent posts. Login first.")
return page, errors.New("Login first")
}
headers := map[string]string{
"Authorization": c.token.AuthToken,
}
body, reqErr := c.client.MakeGetRequest(c.apiUrl+"all", nil, &headers)
if reqErr != nil {
return page, errors.New("HTTP request error")
}
jsonErr := json.Unmarshal(body, &page)
if jsonErr != nil {
log.Fatal(jsonErr)
return page, errors.New("JSON deserialization error")
}
// Move page.Error to response object
if page.Error == "NotAuthorized" {
log.Fatal("Please use correct login and password to fetch recent posts")
return page, errors.New("Not authorized")
}
return page, nil
}
func (c *PointClient) GetNextAllPostsPageBeforeUid(uid int) (Page, error) {
var nextPage Page
if len(c.token.AuthToken) == 0 {
log.Fatal("Can not get recent posts. Login first.")
return nextPage, errors.New("Login first")
}
data := url.Values{}
data.Set("before", strconv.Itoa(uid))
headers := map[string]string{
"Authorization": c.token.AuthToken,
}
body, req_err := c.client.MakeGetRequest(c.apiUrl+"all", &data, &headers)
if req_err != nil {
return nextPage, errors.New("HTTP request error")
}
json_err := json.Unmarshal(body, &nextPage)
if json_err != nil {
log.Fatal(json_err)
return nextPage, errors.New("JSON deserialization error")
}
// Move page.Error to response object
if nextPage.Error == "NotAuthorized" {
log.Fatal("Please use correct login and password to fetch recent posts")
return nextPage, errors.New("Not authorized")
}
return nextPage, nil
}
func (c *PointClient) GetNextAllPostsPage(page Page) (Page, error) {
var nextPage Page
if len(page.Posts) == 0 {
return nextPage, errors.New("Page must have has_next=true and some posts")
}
if len(c.token.AuthToken) == 0 {
log.Fatal("Can not get recent posts. Login first.")
return nextPage, errors.New("Login first")
}
nextPage, err := c.GetNextAllPostsPageBeforeUid(page.Posts[len(page.Posts)-1].Uid)
return nextPage, err
}

10
point/meta_post.go
View file

@ -1,10 +0,0 @@
package point
type MetaPost struct {
Bookmarked bool `json:"bookmarked"`
Uid int `json:"uid"`
Subscribed bool `json:"subscribed"`
Editable bool `json:"editable"`
Recommended bool `json:"recommended"`
Post Post `json:"post"`
}

7
point/page.go
View file

@ -1,7 +0,0 @@
package point
type Page struct {
HasNext bool `json:"has_next"`
Posts []MetaPost `json:"posts"`
Error string `json:"error"`
}

13
point/post.go
View file

@ -1,13 +0,0 @@
package point
type Post struct {
Id string `json:"id"`
Author User `json:"author"`
Tags []string `json:"tags"`
Text string `json:"text"`
Files []string `json:"files"`
CommentsCount int `json:"comments_count"`
Created string `json:"created"`
Pinned bool `json:"pinned"`
Private bool `json:"private"`
}

7
point/token.go
View file

@ -1,7 +0,0 @@
package point
type Token struct {
AuthToken string `json:"token"`
CsRfToken string `json:"csrf_token"`
Error string `json:"error"`
}

6
point/user.go
View file

@ -1,6 +0,0 @@
package point
type User struct {
Id int `json:"id"`
Login string `json:"login"`
}

8
point_post_crawler.go
View file

@ -1,8 +1,8 @@
package main package main
import ( import (
"bitbucket.org/skobkin/point-tools-crawler/point" "bitbucket.org/skobkin/point-api-go"
"bitbucket.org/skobkin/point-tools-crawler/point_tools" "bitbucket.org/skobkin/point-tools-go"
"flag" "flag"
"fmt" "fmt"
"log" "log"
@ -37,8 +37,8 @@ func main() {
return return
} }
pointClient := point.NewClient(pointApiUrl) pointClient := point.New(pointApiUrl)
pointToolsClient := point_tools.NewClient(pointToolsApiUrl, pointToolsToken) pointToolsClient := point_tools.New(pointToolsApiUrl, pointToolsToken)
_, loginErr := pointClient.Login(pointLogin, pointPassword) _, loginErr := pointClient.Login(pointLogin, pointPassword)

6
point_tools/api_error.go
View file

@ -1,6 +0,0 @@
package point_tools
type ApiError struct {
Code int `json:"code"`
Message string `json:"message"`
}

63
point_tools/client.go
View file

@ -1,63 +0,0 @@
package point_tools
import (
"bitbucket.org/skobkin/point-tools-crawler/point"
"bitbucket.org/skobkin/point-tools-crawler/simple_http"
"encoding/json"
"log"
"net/http"
"net/url"
)
type PointToolsClient struct {
client simple_http.Client
apiUrl string
token string
}
func NewClient(apiUrl, token string) PointToolsClient {
return PointToolsClient{
simple_http.Client{http.Client{}},
apiUrl,
token,
}
}
func GetPageJSON(page point.Page) (string, error) {
b, err := json.Marshal(page)
if err != nil {
log.Println("point.Page serialize error:", err)
return "", err
}
return string(b[:]), nil
}
func (c *PointToolsClient) SendPage(page point.Page) (ReceiveAllPageResponse, error) {
var response ReceiveAllPageResponse
jsonStr, err := GetPageJSON(page)
if err != nil {
return response, err
}
data := url.Values{}
data.Set("token", c.token)
data.Add("json", jsonStr)
body, reqErr := c.client.MakePostRequest(c.apiUrl+"all/page", data, nil)
if reqErr != nil {
return response, reqErr
}
jsonErr := json.Unmarshal(body, &response)
if jsonErr != nil {
return response, jsonErr
}
return response, nil
}

16
point_tools/receive_all_page_response.go
View file

@ -1,16 +0,0 @@
package point_tools
const (
STATUS_SUCCESS string = "success"
STATUS_FAILURE string = "fail"
)
type ReceiveAllPageResponse struct {
Status string `json:"status"`
Data ReceiveAllPageData `json:"data"`
Error ApiError `json:"error"`
}
type ReceiveAllPageData struct {
Continue bool `json:"continue"`
}

97
simple_http/client.go
View file

@ -1,97 +0,0 @@
package simple_http
import (
"io/ioutil"
"log"
"net/http"
"net/url"
"strings"
)
type Client struct {
HttpClient http.Client
}
func (c *Client) getResponseBody(resp *http.Response) ([]byte, error) {
body, readErr := ioutil.ReadAll(resp.Body)
if readErr != nil {
log.Fatal(readErr)
return nil, readErr
}
return body, nil
}
func (c *Client) MakeGetRequest(url string, params *url.Values, headers *map[string]string) ([]byte, error) {
var body []byte
if params != nil {
url += "?" + params.Encode()
}
req, reqErr := http.NewRequest("GET", url, nil)
if reqErr != nil {
log.Fatal(reqErr)
return body, reqErr
}
// Headers
if headers != nil {
for key, value := range *headers {
req.Header.Set(key, value)
}
}
resp, respErr := c.HttpClient.Do(req)
if respErr != nil {
log.Fatal(respErr)
return nil, respErr
}
defer resp.Body.Close()
body, readErr := c.getResponseBody(resp)
if readErr != nil {
return nil, readErr
}
return body, nil
}
func (c *Client) MakePostRequest(url string, params url.Values, headers *map[string]string) ([]byte, error) {
req, reqErr := http.NewRequest("POST", url, strings.NewReader(params.Encode()))
if reqErr != nil {
log.Fatal(reqErr)
return nil, reqErr
}
// Adding necessary for POST request header
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
// Headers
if headers != nil {
for key, value := range *headers {
req.Header.Set(key, value)
}
}
resp, respErr := c.HttpClient.Do(req)
if respErr != nil {
log.Fatal(respErr)
return nil, respErr
}
body, readErr := c.getResponseBody(resp)
if readErr != nil {
return nil, readErr
}
return body, nil
}