skobkin/point-tools-crawler
1
0
Fork 0
mirror of https://bitbucket.org/skobkin/point-tools-crawler.git synced 2024-12-04 18:25:52 +00:00
Code Issues Projects Releases Wiki Activity

Client now can fetch next pages.

Browse source
This commit is contained in:
Alexey Skobkin 2016-03-16 01:56:51 +03:00
parent 200ff53885
commit 602552baf9
2 changed files with 18 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
point/client.go
View file

@ -8,6 +8,7 @@ import (
"encoding/json"
"errors"
"strings"
"strconv"
)
const point_api_url string = "https://point.im/api/"
@ -167,7 +168,7 @@ func (c *Client) GetNextAllPostsPage(page Page) (Page, error) {
}
data := url.Values{}
data.Add("before", string(page.Posts[len(page.Posts)-1].Uid))
data.Add("before", strconv.Itoa(page.Posts[len(page.Posts)-1].Uid))
body, req_err := c.MakeGetRequest(point_api_url + "all", &data)

19
point_post_crawler.go
View file

@ -8,6 +8,7 @@ import (
"bitbucket.org/skobkin/point_post_crawler/point"
"time"
"log"
"encoding/json"
)
@ -43,24 +44,36 @@ func main() {
}
fmt.Printf("1 page requested\n")
fmt.Printf("%d posts\n", len(page.Posts))
printPageJSON(page)
pageNumber := 1;
for page.HasNext {
pageNumber++
page, req_err := client.GetNextAllPostsPage(page)
page, req_err = client.GetNextAllPostsPage(page)
if req_err != nil {
log.Fatal(req_err)
return
}
printPageJSON(page)
fmt.Printf("%d page requested\n", pageNumber)
fmt.Printf("%d posts\n", len(page.Posts))
time.Sleep(2 * time.Second)
time.Sleep(time.Second)
}
}
func printPageJSON(page point.Page) {
b, err := json.Marshal(page)
if err != nil {
fmt.Println("error:", err)
}
fmt.Println(string(b[:]))
}