diff --git a/point/client.go b/point/client.go new file mode 100644 index 0000000..cfcda48 --- /dev/null +++ b/point/client.go @@ -0,0 +1,192 @@ +package point + +import ( + "log" + "net/http" + "net/url" + "io/ioutil" + "encoding/json" + "errors" + "strings" +) + +const point_api_url string = "https://point.im/api/" + +type Client struct { + token Token +} + +func (c *Client) getResponseBody(resp *http.Response) ([]byte, error) { + body, read_err := ioutil.ReadAll(resp.Body) + + if read_err != nil { + log.Fatal(read_err) + return nil, read_err + } + + return body, nil +} + +func (c *Client) MakeGetRequest(url string, params *url.Values) ([]byte, error) { + var body []byte + + if params != nil { + url += "?" + params.Encode() + } + + req, req_err := http.NewRequest("GET", url, nil) + + if req_err != nil { + log.Fatal(req_err) + return body, req_err + } + + if len(c.token.AuthToken) > 0 { + req.Header.Set("Authorization", c.token.AuthToken) + } + + hc := &http.Client{} + + resp, resp_err := hc.Do(req) + + if resp_err != nil { + log.Fatal(resp_err) + return nil, resp_err + } + + defer resp.Body.Close() + + body, read_err := c.getResponseBody(resp) + + if read_err != nil { + return nil, read_err + } + + return body, nil +} + +func (c *Client) MakePostRequest(url string, params url.Values) ([]byte, error) { + req, req_err := http.NewRequest("POST", url, strings.NewReader(params.Encode())) + + if req_err != nil { + log.Fatal(req_err) + return nil, req_err + } + + hc := &http.Client{} + + if len(c.token.AuthToken) > 0 { + req.Header.Set("Authorization", c.token.AuthToken) + } + + resp, resp_err := hc.Do(req) + + if resp_err != nil { + log.Fatal(resp_err) + return nil, resp_err + } + + body, read_err := c.getResponseBody(resp) + + if read_err != nil { + return nil, read_err + } + + return body, nil +} + +func (c *Client) Login(login, password string) (Token, error) { + var token Token + + data := url.Values{} + data.Add("login", login) + data.Add("password", password) + + body, req_err := c.MakePostRequest(point_api_url + "login", data) + + if req_err != nil { + return token, req_err + } + + json_err := json.Unmarshal(body, &token) + + if json_err != nil { + log.Fatal(json_err) + return token, errors.New("JSON deserialization error") + } + + if token.Error != "" { + return token, errors.New(token.Error) + } + + c.token = token + + return token, nil +} + +func (c *Client) GetRecentAllPostsPage() (Page, error) { + var page Page + + if len(c.token.AuthToken) == 0 { + log.Fatal("Can not get recent posts. Login first.") + return page, errors.New("Login first") + } + + body, req_err := c.MakeGetRequest(point_api_url + "all", nil) + + if req_err != nil { + return page, errors.New("HTTP request error") + } + + json_err := json.Unmarshal(body, &page) + + if json_err != nil { + log.Fatal(json_err) + return page, errors.New("JSON deserialization error") + } + + // Move page.Error to response object + if page.Error == "NotAuthorized" { + log.Fatal("Please use correct login and password to fetch recent posts") + return page, errors.New("Not authorized") + } + + return page, nil +} + +func (c *Client) GetNextAllPostsPage(page Page) (Page, error) { + var nextPage Page + + if false == page.HasNext || len(page.Posts) == 0 { + return nextPage, errors.New("Page must have has_next=true and some posts") + } + + if len(c.token.AuthToken) == 0 { + log.Fatal("Can not get recent posts. Login first.") + return nextPage, errors.New("Login first") + } + + data := url.Values{} + data.Add("before", string(page.Posts[len(page.Posts)-1].Uid)) + + body, req_err := c.MakeGetRequest(point_api_url + "all", &data) + + if req_err != nil { + return nextPage, errors.New("HTTP request error") + } + + json_err := json.Unmarshal(body, &nextPage) + + if json_err != nil { + log.Fatal(json_err) + return nextPage, errors.New("JSON deserialization error") + } + + // Move page.Error to response object + if nextPage.Error == "NotAuthorized" { + log.Fatal("Please use correct login and password to fetch recent posts") + return nextPage, errors.New("Not authorized") + } + + return nextPage, nil +} \ No newline at end of file diff --git a/point/meta_post.go b/point/meta_post.go new file mode 100644 index 0000000..c827bd2 --- /dev/null +++ b/point/meta_post.go @@ -0,0 +1,11 @@ +package point + +type MetaPost struct { + Bookmarked bool `json:"bookmarked"` + Uid int `json:"uid"` + Subscribed bool `json:"subscribed"` + Editable bool `json:"editable"` + Recommended bool `json:"recommended"` + Rec Recommendation `json:"rec"` + Post Post `json:"post"` +} diff --git a/point/page.go b/point/page.go new file mode 100644 index 0000000..8d5b0ee --- /dev/null +++ b/point/page.go @@ -0,0 +1,7 @@ +package point + +type Page struct { + HasNext bool `json:"has_next"` + Posts []MetaPost `json:"posts"` + Error string `json:"error"` +} diff --git a/point/post.go b/point/post.go new file mode 100644 index 0000000..28922ad --- /dev/null +++ b/point/post.go @@ -0,0 +1,12 @@ +package point + +type Post struct { + Id string `json:"id"` + Author User `json:"author"` + Tags [10]string `json:"tags"` + Text string `json:"text"` + CommentsCount int `json:"comments_count"` + Created string `json:"created"` + Pinned bool `json:"pinned"` + Private bool `json:"private"` +} diff --git a/point/recommendation.go b/point/recommendation.go new file mode 100644 index 0000000..f159ad9 --- /dev/null +++ b/point/recommendation.go @@ -0,0 +1,7 @@ +package point + +type Recommendation struct { + Text string `json:"text"` + Comment_id int `json:"comment_id"` + Author User `json:"author"` +} diff --git a/point/token.go b/point/token.go new file mode 100644 index 0000000..26fb9a9 --- /dev/null +++ b/point/token.go @@ -0,0 +1,7 @@ +package point + +type Token struct { + AuthToken string `json:"token"` + CsRfToken string `json:"csrf_token"` + Error string `json:"error"` +} diff --git a/point/user.go b/point/user.go new file mode 100644 index 0000000..ee5a5b3 --- /dev/null +++ b/point/user.go @@ -0,0 +1,6 @@ +package point + +type User struct { + Id int `json:"id"` + Login string `json:"login"` +} diff --git a/point_post_crawler.go b/point_post_crawler.go index 07c5b00..cbb39e0 100644 --- a/point_post_crawler.go +++ b/point_post_crawler.go @@ -1,62 +1,19 @@ package main import ( - "fmt" "flag" + "fmt" //"gopkg.in/yaml.v2" //"net/http" - "encoding/json" + "bitbucket.org/skobkin/point_post_crawler/point" "time" - "net/http" "log" - "io/ioutil" ) -type user struct { - id int - login string -} -type tag struct { - name string -} - -type recommendation struct { - text string - comment_id int - author user -} - -type post struct { - pinned bool - tags [10]int - comments_count int - author user - text string - created string - id string - private bool -} - -type meta_post struct { - bookmarked bool - uid int - subscribed bool - editable bool - recommended bool - rec recommendation - post post -} - -type page struct { - has_next bool - posts [20]meta_post -} - -const point_api_url string = "https://point.im/api/" func main() { - var login, password string; + var login, password string flag.StringVar(&login, "l", "", "Account login") flag.StringVar(&password, "p", "", "Account password") @@ -67,43 +24,43 @@ func main() { return } - // Auth + client := point.Client{} + _, login_err := client.Login(login, password) - // Post processing - page_number := 0; - for { - resp, req_err := http.Get(point_api_url + "recent") + if login_err != nil { + fmt.Printf("Login error %s", login_err) + return + } else { + fmt.Printf("Successfully authenticated!\n") + } + + page, req_err := client.GetRecentAllPostsPage() + + if req_err != nil { + log.Fatal(req_err) + return + } + + fmt.Printf("1 page requested\n") + fmt.Printf("%d posts\n", len(page.Posts)) + + pageNumber := 1; + + for page.HasNext { + pageNumber++ + + page, req_err := client.GetNextAllPostsPage(page) if req_err != nil { log.Fatal(req_err) - continue + return } - defer resp.Body.Close() + fmt.Printf("%d page requested\n", pageNumber) + fmt.Printf("%d posts\n", len(page.Posts)) - body, read_err := ioutil.ReadAll(resp.Body) - - if read_err != nil { - log.Fatal(read_err) - continue - } - - var page page - - json_err := json.Unmarshal(body, &page) - - if json_err != nil { - log.Fatal(json_err) - continue - } - - page_number++ - fmt.Printf("%d done:\n", page_number) - - //fmt.Printf("%+v", body) - fmt.Printf("%s\n", body) - - time.Sleep(10 * time.Second) + time.Sleep(2 * time.Second) } } +