mirror of
https://bitbucket.org/skobkin/point-tools-crawler.git
synced 2024-12-04 02:05:54 +00:00
initial
This commit is contained in:
commit
db03711445
2
.gitignore
vendored
Normal file
2
.gitignore
vendored
Normal file
|
@ -0,0 +1,2 @@
|
|||
/.idea/
|
||||
/bin/
|
109
point_post_crawler.go
Normal file
109
point_post_crawler.go
Normal file
|
@ -0,0 +1,109 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"flag"
|
||||
//"gopkg.in/yaml.v2"
|
||||
//"net/http"
|
||||
"encoding/json"
|
||||
"time"
|
||||
"net/http"
|
||||
"log"
|
||||
"io/ioutil"
|
||||
)
|
||||
|
||||
type user struct {
|
||||
id int
|
||||
login string
|
||||
}
|
||||
|
||||
type tag struct {
|
||||
name string
|
||||
}
|
||||
|
||||
type recommendation struct {
|
||||
text string
|
||||
comment_id int
|
||||
author user
|
||||
}
|
||||
|
||||
type post struct {
|
||||
pinned bool
|
||||
tags [10]int
|
||||
comments_count int
|
||||
author user
|
||||
text string
|
||||
created string
|
||||
id string
|
||||
private bool
|
||||
}
|
||||
|
||||
type meta_post struct {
|
||||
bookmarked bool
|
||||
uid int
|
||||
subscribed bool
|
||||
editable bool
|
||||
recommended bool
|
||||
rec recommendation
|
||||
post post
|
||||
}
|
||||
|
||||
type page struct {
|
||||
has_next bool
|
||||
posts [20]meta_post
|
||||
}
|
||||
|
||||
const point_api_url string = "https://point.im/api/"
|
||||
|
||||
func main() {
|
||||
var login, password string;
|
||||
|
||||
flag.StringVar(&login, "l", "", "Account login")
|
||||
flag.StringVar(&password, "p", "", "Account password")
|
||||
flag.Parse()
|
||||
|
||||
if len(login) < 1 || len(password) < 1 {
|
||||
fmt.Printf("Login and password must be defined")
|
||||
return
|
||||
}
|
||||
|
||||
// Auth
|
||||
|
||||
|
||||
// Post processing
|
||||
page_number := 0;
|
||||
for {
|
||||
resp, req_err := http.Get(point_api_url + "recent")
|
||||
|
||||
if req_err != nil {
|
||||
log.Fatal(req_err)
|
||||
continue
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, read_err := ioutil.ReadAll(resp.Body)
|
||||
|
||||
if read_err != nil {
|
||||
log.Fatal(read_err)
|
||||
continue
|
||||
}
|
||||
|
||||
var page page
|
||||
|
||||
json_err := json.Unmarshal(body, &page)
|
||||
|
||||
if json_err != nil {
|
||||
log.Fatal(json_err)
|
||||
continue
|
||||
}
|
||||
|
||||
page_number++
|
||||
fmt.Printf("%d done:\n", page_number)
|
||||
|
||||
//fmt.Printf("%+v", body)
|
||||
fmt.Printf("%s\n", body)
|
||||
|
||||
time.Sleep(10 * time.Second)
|
||||
}
|
||||
}
|
Loading…
Reference in a new issue