This commit is contained in:
Alexey Skobkin 2016-03-07 23:04:50 +03:00
commit db03711445
2 changed files with 111 additions and 0 deletions

2
.gitignore vendored Normal file
View file

@ -0,0 +1,2 @@
/.idea/
/bin/

109
point_post_crawler.go Normal file
View file

@ -0,0 +1,109 @@
package main
import (
"fmt"
"flag"
//"gopkg.in/yaml.v2"
//"net/http"
"encoding/json"
"time"
"net/http"
"log"
"io/ioutil"
)
type user struct {
id int
login string
}
type tag struct {
name string
}
type recommendation struct {
text string
comment_id int
author user
}
type post struct {
pinned bool
tags [10]int
comments_count int
author user
text string
created string
id string
private bool
}
type meta_post struct {
bookmarked bool
uid int
subscribed bool
editable bool
recommended bool
rec recommendation
post post
}
type page struct {
has_next bool
posts [20]meta_post
}
const point_api_url string = "https://point.im/api/"
func main() {
var login, password string;
flag.StringVar(&login, "l", "", "Account login")
flag.StringVar(&password, "p", "", "Account password")
flag.Parse()
if len(login) < 1 || len(password) < 1 {
fmt.Printf("Login and password must be defined")
return
}
// Auth
// Post processing
page_number := 0;
for {
resp, req_err := http.Get(point_api_url + "recent")
if req_err != nil {
log.Fatal(req_err)
continue
}
defer resp.Body.Close()
body, read_err := ioutil.ReadAll(resp.Body)
if read_err != nil {
log.Fatal(read_err)
continue
}
var page page
json_err := json.Unmarshal(body, &page)
if json_err != nil {
log.Fatal(json_err)
continue
}
page_number++
fmt.Printf("%d done:\n", page_number)
//fmt.Printf("%+v", body)
fmt.Printf("%s\n", body)
time.Sleep(10 * time.Second)
}
}