mirror of
https://bitbucket.org/skobkin/point-tools-crawler.git
synced 2024-11-14 17:53:05 +00:00
Test point API client implementation.
This commit is contained in:
parent
db03711445
commit
200ff53885
192
point/client.go
Normal file
192
point/client.go
Normal file
|
@ -0,0 +1,192 @@
|
|||
package point
|
||||
|
||||
import (
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"io/ioutil"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const point_api_url string = "https://point.im/api/"
|
||||
|
||||
type Client struct {
|
||||
token Token
|
||||
}
|
||||
|
||||
func (c *Client) getResponseBody(resp *http.Response) ([]byte, error) {
|
||||
body, read_err := ioutil.ReadAll(resp.Body)
|
||||
|
||||
if read_err != nil {
|
||||
log.Fatal(read_err)
|
||||
return nil, read_err
|
||||
}
|
||||
|
||||
return body, nil
|
||||
}
|
||||
|
||||
func (c *Client) MakeGetRequest(url string, params *url.Values) ([]byte, error) {
|
||||
var body []byte
|
||||
|
||||
if params != nil {
|
||||
url += "?" + params.Encode()
|
||||
}
|
||||
|
||||
req, req_err := http.NewRequest("GET", url, nil)
|
||||
|
||||
if req_err != nil {
|
||||
log.Fatal(req_err)
|
||||
return body, req_err
|
||||
}
|
||||
|
||||
if len(c.token.AuthToken) > 0 {
|
||||
req.Header.Set("Authorization", c.token.AuthToken)
|
||||
}
|
||||
|
||||
hc := &http.Client{}
|
||||
|
||||
resp, resp_err := hc.Do(req)
|
||||
|
||||
if resp_err != nil {
|
||||
log.Fatal(resp_err)
|
||||
return nil, resp_err
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, read_err := c.getResponseBody(resp)
|
||||
|
||||
if read_err != nil {
|
||||
return nil, read_err
|
||||
}
|
||||
|
||||
return body, nil
|
||||
}
|
||||
|
||||
func (c *Client) MakePostRequest(url string, params url.Values) ([]byte, error) {
|
||||
req, req_err := http.NewRequest("POST", url, strings.NewReader(params.Encode()))
|
||||
|
||||
if req_err != nil {
|
||||
log.Fatal(req_err)
|
||||
return nil, req_err
|
||||
}
|
||||
|
||||
hc := &http.Client{}
|
||||
|
||||
if len(c.token.AuthToken) > 0 {
|
||||
req.Header.Set("Authorization", c.token.AuthToken)
|
||||
}
|
||||
|
||||
resp, resp_err := hc.Do(req)
|
||||
|
||||
if resp_err != nil {
|
||||
log.Fatal(resp_err)
|
||||
return nil, resp_err
|
||||
}
|
||||
|
||||
body, read_err := c.getResponseBody(resp)
|
||||
|
||||
if read_err != nil {
|
||||
return nil, read_err
|
||||
}
|
||||
|
||||
return body, nil
|
||||
}
|
||||
|
||||
func (c *Client) Login(login, password string) (Token, error) {
|
||||
var token Token
|
||||
|
||||
data := url.Values{}
|
||||
data.Add("login", login)
|
||||
data.Add("password", password)
|
||||
|
||||
body, req_err := c.MakePostRequest(point_api_url + "login", data)
|
||||
|
||||
if req_err != nil {
|
||||
return token, req_err
|
||||
}
|
||||
|
||||
json_err := json.Unmarshal(body, &token)
|
||||
|
||||
if json_err != nil {
|
||||
log.Fatal(json_err)
|
||||
return token, errors.New("JSON deserialization error")
|
||||
}
|
||||
|
||||
if token.Error != "" {
|
||||
return token, errors.New(token.Error)
|
||||
}
|
||||
|
||||
c.token = token
|
||||
|
||||
return token, nil
|
||||
}
|
||||
|
||||
func (c *Client) GetRecentAllPostsPage() (Page, error) {
|
||||
var page Page
|
||||
|
||||
if len(c.token.AuthToken) == 0 {
|
||||
log.Fatal("Can not get recent posts. Login first.")
|
||||
return page, errors.New("Login first")
|
||||
}
|
||||
|
||||
body, req_err := c.MakeGetRequest(point_api_url + "all", nil)
|
||||
|
||||
if req_err != nil {
|
||||
return page, errors.New("HTTP request error")
|
||||
}
|
||||
|
||||
json_err := json.Unmarshal(body, &page)
|
||||
|
||||
if json_err != nil {
|
||||
log.Fatal(json_err)
|
||||
return page, errors.New("JSON deserialization error")
|
||||
}
|
||||
|
||||
// Move page.Error to response object
|
||||
if page.Error == "NotAuthorized" {
|
||||
log.Fatal("Please use correct login and password to fetch recent posts")
|
||||
return page, errors.New("Not authorized")
|
||||
}
|
||||
|
||||
return page, nil
|
||||
}
|
||||
|
||||
func (c *Client) GetNextAllPostsPage(page Page) (Page, error) {
|
||||
var nextPage Page
|
||||
|
||||
if false == page.HasNext || len(page.Posts) == 0 {
|
||||
return nextPage, errors.New("Page must have has_next=true and some posts")
|
||||
}
|
||||
|
||||
if len(c.token.AuthToken) == 0 {
|
||||
log.Fatal("Can not get recent posts. Login first.")
|
||||
return nextPage, errors.New("Login first")
|
||||
}
|
||||
|
||||
data := url.Values{}
|
||||
data.Add("before", string(page.Posts[len(page.Posts)-1].Uid))
|
||||
|
||||
body, req_err := c.MakeGetRequest(point_api_url + "all", &data)
|
||||
|
||||
if req_err != nil {
|
||||
return nextPage, errors.New("HTTP request error")
|
||||
}
|
||||
|
||||
json_err := json.Unmarshal(body, &nextPage)
|
||||
|
||||
if json_err != nil {
|
||||
log.Fatal(json_err)
|
||||
return nextPage, errors.New("JSON deserialization error")
|
||||
}
|
||||
|
||||
// Move page.Error to response object
|
||||
if nextPage.Error == "NotAuthorized" {
|
||||
log.Fatal("Please use correct login and password to fetch recent posts")
|
||||
return nextPage, errors.New("Not authorized")
|
||||
}
|
||||
|
||||
return nextPage, nil
|
||||
}
|
11
point/meta_post.go
Normal file
11
point/meta_post.go
Normal file
|
@ -0,0 +1,11 @@
|
|||
package point
|
||||
|
||||
type MetaPost struct {
|
||||
Bookmarked bool `json:"bookmarked"`
|
||||
Uid int `json:"uid"`
|
||||
Subscribed bool `json:"subscribed"`
|
||||
Editable bool `json:"editable"`
|
||||
Recommended bool `json:"recommended"`
|
||||
Rec Recommendation `json:"rec"`
|
||||
Post Post `json:"post"`
|
||||
}
|
7
point/page.go
Normal file
7
point/page.go
Normal file
|
@ -0,0 +1,7 @@
|
|||
package point
|
||||
|
||||
type Page struct {
|
||||
HasNext bool `json:"has_next"`
|
||||
Posts []MetaPost `json:"posts"`
|
||||
Error string `json:"error"`
|
||||
}
|
12
point/post.go
Normal file
12
point/post.go
Normal file
|
@ -0,0 +1,12 @@
|
|||
package point
|
||||
|
||||
type Post struct {
|
||||
Id string `json:"id"`
|
||||
Author User `json:"author"`
|
||||
Tags [10]string `json:"tags"`
|
||||
Text string `json:"text"`
|
||||
CommentsCount int `json:"comments_count"`
|
||||
Created string `json:"created"`
|
||||
Pinned bool `json:"pinned"`
|
||||
Private bool `json:"private"`
|
||||
}
|
7
point/recommendation.go
Normal file
7
point/recommendation.go
Normal file
|
@ -0,0 +1,7 @@
|
|||
package point
|
||||
|
||||
type Recommendation struct {
|
||||
Text string `json:"text"`
|
||||
Comment_id int `json:"comment_id"`
|
||||
Author User `json:"author"`
|
||||
}
|
7
point/token.go
Normal file
7
point/token.go
Normal file
|
@ -0,0 +1,7 @@
|
|||
package point
|
||||
|
||||
type Token struct {
|
||||
AuthToken string `json:"token"`
|
||||
CsRfToken string `json:"csrf_token"`
|
||||
Error string `json:"error"`
|
||||
}
|
6
point/user.go
Normal file
6
point/user.go
Normal file
|
@ -0,0 +1,6 @@
|
|||
package point
|
||||
|
||||
type User struct {
|
||||
Id int `json:"id"`
|
||||
Login string `json:"login"`
|
||||
}
|
|
@ -1,62 +1,19 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"flag"
|
||||
"fmt"
|
||||
//"gopkg.in/yaml.v2"
|
||||
//"net/http"
|
||||
"encoding/json"
|
||||
"bitbucket.org/skobkin/point_post_crawler/point"
|
||||
"time"
|
||||
"net/http"
|
||||
"log"
|
||||
"io/ioutil"
|
||||
)
|
||||
|
||||
type user struct {
|
||||
id int
|
||||
login string
|
||||
}
|
||||
|
||||
type tag struct {
|
||||
name string
|
||||
}
|
||||
|
||||
type recommendation struct {
|
||||
text string
|
||||
comment_id int
|
||||
author user
|
||||
}
|
||||
|
||||
type post struct {
|
||||
pinned bool
|
||||
tags [10]int
|
||||
comments_count int
|
||||
author user
|
||||
text string
|
||||
created string
|
||||
id string
|
||||
private bool
|
||||
}
|
||||
|
||||
type meta_post struct {
|
||||
bookmarked bool
|
||||
uid int
|
||||
subscribed bool
|
||||
editable bool
|
||||
recommended bool
|
||||
rec recommendation
|
||||
post post
|
||||
}
|
||||
|
||||
type page struct {
|
||||
has_next bool
|
||||
posts [20]meta_post
|
||||
}
|
||||
|
||||
const point_api_url string = "https://point.im/api/"
|
||||
|
||||
func main() {
|
||||
var login, password string;
|
||||
var login, password string
|
||||
|
||||
flag.StringVar(&login, "l", "", "Account login")
|
||||
flag.StringVar(&password, "p", "", "Account password")
|
||||
|
@ -67,43 +24,43 @@ func main() {
|
|||
return
|
||||
}
|
||||
|
||||
// Auth
|
||||
client := point.Client{}
|
||||
|
||||
_, login_err := client.Login(login, password)
|
||||
|
||||
// Post processing
|
||||
page_number := 0;
|
||||
for {
|
||||
resp, req_err := http.Get(point_api_url + "recent")
|
||||
if login_err != nil {
|
||||
fmt.Printf("Login error %s", login_err)
|
||||
return
|
||||
} else {
|
||||
fmt.Printf("Successfully authenticated!\n")
|
||||
}
|
||||
|
||||
page, req_err := client.GetRecentAllPostsPage()
|
||||
|
||||
if req_err != nil {
|
||||
log.Fatal(req_err)
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Printf("1 page requested\n")
|
||||
fmt.Printf("%d posts\n", len(page.Posts))
|
||||
|
||||
pageNumber := 1;
|
||||
|
||||
for page.HasNext {
|
||||
pageNumber++
|
||||
|
||||
page, req_err := client.GetNextAllPostsPage(page)
|
||||
|
||||
if req_err != nil {
|
||||
log.Fatal(req_err)
|
||||
continue
|
||||
return
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
fmt.Printf("%d page requested\n", pageNumber)
|
||||
fmt.Printf("%d posts\n", len(page.Posts))
|
||||
|
||||
body, read_err := ioutil.ReadAll(resp.Body)
|
||||
|
||||
if read_err != nil {
|
||||
log.Fatal(read_err)
|
||||
continue
|
||||
}
|
||||
|
||||
var page page
|
||||
|
||||
json_err := json.Unmarshal(body, &page)
|
||||
|
||||
if json_err != nil {
|
||||
log.Fatal(json_err)
|
||||
continue
|
||||
}
|
||||
|
||||
page_number++
|
||||
fmt.Printf("%d done:\n", page_number)
|
||||
|
||||
//fmt.Printf("%+v", body)
|
||||
fmt.Printf("%s\n", body)
|
||||
|
||||
time.Sleep(10 * time.Second)
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in a new issue